# `checks.module-eval-bao-basics` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup baoNames baoSettings baoStream bridgePorts ; baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; }; baoShamir = hive { deploy.bao.enable = true; deploy.bao.seal = "shamir"; }; baoExplicitCerts = hive { deploy.bao.enable = true; deploy.bao.serverCertFile = "/etc/pki/bao.pem"; deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem"; }; autheliaOnly = hive { deploy.authelia.enable = true; }; baoWithForge = hive { deploy.bao.enable = true; deploy.forgejo.enable = true; }; # The store's units live inside its container, so the gates below have to # look there rather than at the host's service set. baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; tlsDir = "/var/lib/swarm-bao-tls"; cases = [ { # The store's seal is spread over six gates — the stanza, the # provisioning unit, two bind mounts, a device and an EnvironmentFile. # Rendering only some of them is the dangerous state: a store that # says hardware-backed and seals with a software key, which no # assertion can catch because every value is individually valid. name = "a shamir store renders no TPM provisioning unit"; ok = !(baoUnits baoShamir ? swarm-bao-token); } { # Presence control for the case above. Without it, a typo in the # option name would satisfy the absence arm forever. The second half is # the fix itself: the unit has to run where openbao's `DynamicUser` is # allocated, and a host unit writing the same bytes has no name to hand # them to. name = "a pkcs11 store provisions the token in the container, not on the host"; ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token); } { # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn # mounts its own /dev and cannot create device nodes, so permission to # use a device that was never bound in opens nothing. Neither half # fails on its own, which is why they are asserted as a pair. name = "a pkcs11 store gets the TPM device bound in, not merely allowed"; ok = let c = baoPkcs11.containers.swarm-bao; in (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices; } { # The stanza's label and the label the unit creates are two literals that # have to name one object, and the mechanism is asserted with them # because it is valid only for an RSA key: openbao takes AES-GCM or # RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the # seal asked for. Every wrong combination renders and deploys, and # surfaces as a pkcs11 error at `operator init`. name = "the seal asks for the RSA key the provisioning unit creates"; ok = let p = (baoSettings baoPkcs11).seal.pkcs11; in (p.mechanism or "") == "CKM_RSA_PKCS_OAEP" && lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" ( (baoUnits baoPkcs11).swarm-bao-token.script or "" ); } { # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is # read-only to the seal however it is owned, and the group is the only # handle on a uid allocated at start. Dropping either surfaces as a # pkcs11 error deep in a library, naming neither the mount nor the user. name = "the store's seal may write the token directory, and is in both its groups"; ok = let sc = (baoUnits baoPkcs11).openbao.serviceConfig; in # `or [ ]` rather than a bare select: the interesting mutation is the # key being gone, and a select would abort the whole run with a nix # trace instead of failing this case by name. builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]) && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]); } { # The device node belongs to the HOST and is matched by NUMBER, while the # unit that opens it lives in the container — so the two sides holding # the same gid is the entire mechanism. Letting either side auto-allocate # renders cleanly, deploys cleanly, and leaves a 0660 node the seal # cannot open. Compared rather than each checked against a literal: the # property is that they AGREE, not what they agree on. name = "the TPM group has the same gid on the host and inside the container"; ok = let host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null; inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null; in host != null && host == inner; } { # Absence arm for the case above — a shamir store never opens a TPM, so # it must not claim a device node's group. Without this, pinning the gid # unconditionally would look identical. name = "a shamir store claims no TPM device group"; ok = !(baoShamir.users.groups ? swarm-bao-tpm); } { # The store's mTLS identity is a separate trust domain from both CAs in # this tree, because it must not come from an authority the store will # itself distribute. What supplies it is the glue, which mints a CA of # the store's own — so an enabled store has all three paths, and if this # ever reads null again the store stops coming up on its own. name = "a deployed store is given its own certificate, key and client CA"; ok = let b = baoPkcs11.services.hyperhive.deploy.bao; in b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null; } { # Everything the glue sets is `mkDefault`, and this is the case that # says so: a deployment whose certificates come from somewhere the glue # has never heard of must win. Also the presence control for the case # above — a renamed option would read `null` on both and satisfy # neither, but only this one names a value. name = "an operator's own certificate path beats the glue's default"; ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem"; } { # Absence arm. A store with nothing to serve renders no reader, so the # unit is a function of the PAIRING rather than of the store — which is # the property that makes it glue instead of a feature of either side. name = "a store with no homeserver beside it renders no token reader"; ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token); } { # The name a reader dials has to resolve where the store runs; a # multi-host swarm resolves it upstream instead. name = "the store's host answers for the store's name"; ok = builtins.elem "bao.t.local" (baoNames baoPkcs11); } { # What makes that name reachable from inside an agent container, and the # single reason it is a stream server rather than a vhost: `ssl_preread` # routes on the SNI without decrypting, so the handshake bao completes is # still the client's own and the certificate it authenticates by arrives # intact. Terminating here would hand the store one identity for the # whole swarm. name = "the store's host passes connections through without terminating TLS"; ok = let s = baoStream baoPkcs11; in lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s; } { # The address half, and it is bridge-only for a reason a wildcard would # hide until deploy: the store already holds `127.0.0.1:8200` in this # same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start # — taking every hive domain behind the gateway down with it. name = "the passthrough listens on the bridge, not on every address"; ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11); } { # The routing half: the SNI picks the backend and the only name that # resolves to one is the store's own. A `default` that pointed anywhere # would make this host a relay for whatever name a client invented. name = "the passthrough routes only the store's name, to its loopback listener"; ok = let s = baoStream baoPkcs11; in lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s && lib.hasInfix "bao.t.local 127.0.0.1:8200;" s && lib.hasInfix ''default "";'' s; } { # The listener is only half of reachable: the bridge firewall drops # everything not named here, and a silent drop is the failure that reads # as "the store is down" from inside a container. name = "the store's port is open on the bridge where the store runs"; ok = builtins.elem 8200 (bridgePorts baoPkcs11); } { # Raft refuses to start without it, and says so in a message that names # neither the setting nor the stanza. name = "the store advertises a cluster address"; ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or ""); } { # The listener trusts the agent CA through a file no cert-auth role # names. At least one listener verifies clients, so an empty set cannot # pass. name = "every client-verifying listener reads the listener-only bundle"; ok = let verifying = lib.filter (l: l ? tls_client_ca_file) ( lib.attrValues (baoSettings baoPkcs11).listener ); in verifying != [ ] && lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying; } { # The other half of keeping agents out of host roles: those roles pin # the store CA alone. The positive count is the control that the text # searched is the one the roles are written in. name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA"; ok = let scripts = lib.concatStrings ( lib.mapAttrsToList ( n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "") ) baoPkcs11.systemd.services ); in lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts && !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts) && !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts); } { # Both writers compose the bundle through one function, which refuses # a result that does not begin with the store CA and replaces the file # only when its bytes change. name = "both bundle writers use the one composer, which keeps the store CA first"; ok = let s = baoPkcs11.systemd.services; composes = u: lib.hasInfix "compose_listener_bundle() {" u.script && lib.hasInfix "\ncompose_listener_bundle\n" u.script && lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script && lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script && lib.hasInfix "mktemp -p ${tlsDir} " u.script; in s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki; } { # openbao reads its client-CA file only at start, so picking up the # agent CA is a restart: automatic where the store unseals itself, # printed where a human has to. The shamir arm is the control. name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir"; ok = let restart = ''systemctl --machine="$machine" restart openbao.service''; p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script; sh = baoShamir.systemd.services.swarm-bao-agent-pki.script; in lib.hasInfix restart p && lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p && !(lib.hasInfix restart sh) && lib.hasInfix "# then unseal" sh; } { # The browser UI's listener must not loosen the one every reader dials. # `loopback` is named so an empty filter cannot pass. name = "every listener but metrics and the UI still requires a client certificate"; ok = let l = (baoSettings baoPkcs11).listener; api = builtins.removeAttrs l [ "metrics" "ui" ]; in api ? loopback && lib.all (x: (x.tls_require_and_verify_client_cert or false) == true) (lib.attrValues api); } { # No client certificate on this one, so loopback is all that keeps it # away from everything but this netns. name = "the UI listener is loopback-only and the UI is served"; ok = let s = baoSettings baoPkcs11; in (s.ui or false) == true && lib.hasPrefix "127.0.0.1:" (s.listener.ui.address or "") && !(s.listener.ui ? tls_require_and_verify_client_cert); } { # The nginx in front of it is the listener's only client and must bind # loopback too, and it refuses the endpoints a browser never needs. name = "the UI's nginx binds loopback and refuses the unseal and root-generation paths"; ok = let v = baoPkcs11.containers.swarm-bao.config.services.nginx.virtualHosts.bao-ui; in lib.all (x: x.addr == "127.0.0.1") v.listen && (v.locations."~* ^/v1/sys/(unseal|seal|step-down)/?$".return or null) == "403" && (v.locations."~* ^/v1/sys/(rekey|generate-root)".return or null) == "403"; } { # A new vhost gets no group gate from `auth_request`; authelia's # default policy admits any session. Evaluated on a host that runs # authelia and no store, because that is where the rule has to render. name = "authelia restricts the store's UI name to admins wherever authelia runs"; ok = let rules = autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules; in !autheliaOnly.services.hyperhive.deploy.bao.enable && builtins.elem { domain = "bao-ui.t.local"; subject = [ "group:admins" ]; policy = "one_factor"; } rules; } { # The other half of that rule: the vhost exists, and asks authelia. name = "the store's host serves the UI name behind the authelia subrequest"; ok = let v = baoPkcs11.services.nginx.virtualHosts."bao-ui.t.local"; in lib.hasInfix "auth_request /__hive_authelia;" (v.locations."/".extraConfig or "") && (v.locations."/".proxyPass or "") == "http://127.0.0.1:8206"; } { # The stream passthrough every reader dials rides the same nginx as the # UI's vhost. On a host whose only swarm name is the UI, that nginx must # not wait on the services leaf, which needs a login to this very store. # The control is a store host that also fronts forge: its gateway waits, # exactly as it did before the UI existed. name = "a store host fronting only the UI does not hold nginx on a store login"; ok = let waits = m: builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy; orders = m: builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before; in baoPkcs11.services.hyperhive.swarm.localServiceDomains == [ "bao-ui.t.local" ] && baoPkcs11.systemd.services ? hive-gateway-self-signed-cert && !(waits baoPkcs11) && !(orders baoPkcs11) && waits baoWithForge && orders baoWithForge; } ]; in runGroup "bao-basics" cases