3898ca33c7
openbao gains a second listener, `ui`, on 127.0.0.1:<deploy.bao.uiPort> (default 8204) with TLS off and no client-certificate requirement, and `ui = true`. The existing listeners are unchanged. An nginx inside the store's container, on 127.0.0.1:<deploy.bao.uiProxyPort> (default 8206), forwards only /ui/ and /v1/ to it, redirects / to /ui/, answers 403 on sys/unseal, sys/seal, sys/step-down, sys/rekey* and sys/generate-root*, and 404 on everything else. The gateway on the store's host serves `swarm.bao.ui.domain` (default bao-ui.<swarm>) behind the authelia auth_request subrequest, proxying to that nginx; the name joins serviceDomains and localNames like every other gateway-published swarm service. authelia gets an access_control rule restricting that name to group:admins, rendered wherever authelia runs, since the default policy admits any session. Trade-off, ruled by the operator on the parent issue: the UI listener asks for no client certificate, so on that door a bao token alone is the credential. Three comments and a doc line claimed every API listener demands a client certificate; they now except the loopback UI listener. The module-eval case counting declared listeners excludes `ui` by name, as it already did `metrics`. On a self-signed gateway, the UI's name is a swarm service name, so its host requests the services leaf from the store. `swarm-services-cert` sits Before= and RequiredBy= the gateway's cert import, which nginx Requires=. On a host whose only swarm name is the UI, that would hold nginx, and with it the stream passthrough every reader dials, on a login to a store that may be sealed. hive-tls drops those two edges exactly when the UI is the only local swarm name: nginx starts on the existing hive-leaf fallback, and the script's existing re-import reloads nginx once the leaf issues. Every other host keeps both edges.
382 lines
16 KiB
Nix
382 lines
16 KiB
Nix
# `checks.module-eval-bao-basics` — see ./lib.nix for the shared
|
|
# rationale (why this suite exists, naming convention, "evaluates
|
|
# not executes").
|
|
{
|
|
pkgs,
|
|
lib,
|
|
self,
|
|
nixosSystem,
|
|
}:
|
|
let
|
|
inherit
|
|
(import ./lib.nix {
|
|
inherit
|
|
pkgs
|
|
lib
|
|
self
|
|
nixosSystem
|
|
;
|
|
})
|
|
hive
|
|
runGroup
|
|
baoNames
|
|
baoSettings
|
|
baoStream
|
|
bridgePorts
|
|
;
|
|
|
|
baoPkcs11 = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "pkcs11";
|
|
};
|
|
|
|
baoShamir = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.seal = "shamir";
|
|
};
|
|
|
|
baoExplicitCerts = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.bao.serverCertFile = "/etc/pki/bao.pem";
|
|
deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem";
|
|
};
|
|
|
|
autheliaOnly = hive { deploy.authelia.enable = true; };
|
|
|
|
baoWithForge = hive {
|
|
deploy.bao.enable = true;
|
|
deploy.forgejo.enable = true;
|
|
};
|
|
|
|
# The store's units live inside its container, so the gates below have to
|
|
# look there rather than at the host's service set.
|
|
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
|
|
|
|
tlsDir = "/var/lib/swarm-bao-tls";
|
|
cases = [
|
|
{
|
|
# The store's seal is spread over six gates — the stanza, the
|
|
# provisioning unit, two bind mounts, a device and an EnvironmentFile.
|
|
# Rendering only some of them is the dangerous state: a store that
|
|
# says hardware-backed and seals with a software key, which no
|
|
# assertion can catch because every value is individually valid.
|
|
name = "a shamir store renders no TPM provisioning unit";
|
|
ok = !(baoUnits baoShamir ? swarm-bao-token);
|
|
}
|
|
{
|
|
# Presence control for the case above. Without it, a typo in the
|
|
# option name would satisfy the absence arm forever. The second half is
|
|
# the fix itself: the unit has to run where openbao's `DynamicUser` is
|
|
# allocated, and a host unit writing the same bytes has no name to hand
|
|
# them to.
|
|
name = "a pkcs11 store provisions the token in the container, not on the host";
|
|
ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token);
|
|
}
|
|
{
|
|
# `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn
|
|
# mounts its own /dev and cannot create device nodes, so permission to
|
|
# use a device that was never bound in opens nothing. Neither half
|
|
# fails on its own, which is why they are asserted as a pair.
|
|
name = "a pkcs11 store gets the TPM device bound in, not merely allowed";
|
|
ok =
|
|
let
|
|
c = baoPkcs11.containers.swarm-bao;
|
|
in
|
|
(c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices;
|
|
}
|
|
{
|
|
# The stanza's label and the label the unit creates are two literals that
|
|
# have to name one object, and the mechanism is asserted with them
|
|
# because it is valid only for an RSA key: openbao takes AES-GCM or
|
|
# RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the
|
|
# seal asked for. Every wrong combination renders and deploys, and
|
|
# surfaces as a pkcs11 error at `operator init`.
|
|
name = "the seal asks for the RSA key the provisioning unit creates";
|
|
ok =
|
|
let
|
|
p = (baoSettings baoPkcs11).seal.pkcs11;
|
|
in
|
|
(p.mechanism or "") == "CKM_RSA_PKCS_OAEP"
|
|
&& lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" (
|
|
(baoUnits baoPkcs11).swarm-bao-token.script or ""
|
|
);
|
|
}
|
|
{
|
|
# `DynamicUser` implies `ProtectSystem=strict`, so the token directory is
|
|
# read-only to the seal however it is owned, and the group is the only
|
|
# handle on a uid allocated at start. Dropping either surfaces as a
|
|
# pkcs11 error deep in a library, naming neither the mount nor the user.
|
|
name = "the store's seal may write the token directory, and is in both its groups";
|
|
ok =
|
|
let
|
|
sc = (baoUnits baoPkcs11).openbao.serviceConfig;
|
|
in
|
|
# `or [ ]` rather than a bare select: the interesting mutation is the
|
|
# key being gone, and a select would abort the whole run with a nix
|
|
# trace instead of failing this case by name.
|
|
builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ])
|
|
&& builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ])
|
|
&& builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]);
|
|
}
|
|
{
|
|
# The device node belongs to the HOST and is matched by NUMBER, while the
|
|
# unit that opens it lives in the container — so the two sides holding
|
|
# the same gid is the entire mechanism. Letting either side auto-allocate
|
|
# renders cleanly, deploys cleanly, and leaves a 0660 node the seal
|
|
# cannot open. Compared rather than each checked against a literal: the
|
|
# property is that they AGREE, not what they agree on.
|
|
name = "the TPM group has the same gid on the host and inside the container";
|
|
ok =
|
|
let
|
|
host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null;
|
|
inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null;
|
|
in
|
|
host != null && host == inner;
|
|
}
|
|
{
|
|
# Absence arm for the case above — a shamir store never opens a TPM, so
|
|
# it must not claim a device node's group. Without this, pinning the gid
|
|
# unconditionally would look identical.
|
|
name = "a shamir store claims no TPM device group";
|
|
ok = !(baoShamir.users.groups ? swarm-bao-tpm);
|
|
}
|
|
{
|
|
# The store's mTLS identity is a separate trust domain from both CAs in
|
|
# this tree, because it must not come from an authority the store will
|
|
# itself distribute. What supplies it is the glue, which mints a CA of
|
|
# the store's own — so an enabled store has all three paths, and if this
|
|
# ever reads null again the store stops coming up on its own.
|
|
name = "a deployed store is given its own certificate, key and client CA";
|
|
ok =
|
|
let
|
|
b = baoPkcs11.services.hyperhive.deploy.bao;
|
|
in
|
|
b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null;
|
|
}
|
|
{
|
|
# Everything the glue sets is `mkDefault`, and this is the case that
|
|
# says so: a deployment whose certificates come from somewhere the glue
|
|
# has never heard of must win. Also the presence control for the case
|
|
# above — a renamed option would read `null` on both and satisfy
|
|
# neither, but only this one names a value.
|
|
name = "an operator's own certificate path beats the glue's default";
|
|
ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem";
|
|
}
|
|
{
|
|
# Absence arm. A store with nothing to serve renders no reader, so the
|
|
# unit is a function of the PAIRING rather than of the store — which is
|
|
# the property that makes it glue instead of a feature of either side.
|
|
name = "a store with no homeserver beside it renders no token reader";
|
|
ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token);
|
|
}
|
|
{
|
|
# The name a reader dials has to resolve where the store runs; a
|
|
# multi-host swarm resolves it upstream instead.
|
|
name = "the store's host answers for the store's name";
|
|
ok = builtins.elem "bao.t.local" (baoNames baoPkcs11);
|
|
}
|
|
{
|
|
# What makes that name reachable from inside an agent container, and the
|
|
# single reason it is a stream server rather than a vhost: `ssl_preread`
|
|
# routes on the SNI without decrypting, so the handshake bao completes is
|
|
# still the client's own and the certificate it authenticates by arrives
|
|
# intact. Terminating here would hand the store one identity for the
|
|
# whole swarm.
|
|
name = "the store's host passes connections through without terminating TLS";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s;
|
|
}
|
|
{
|
|
# The address half, and it is bridge-only for a reason a wildcard would
|
|
# hide until deploy: the store already holds `127.0.0.1:8200` in this
|
|
# same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start
|
|
# — taking every hive domain behind the gateway down with it.
|
|
name = "the passthrough listens on the bridge, not on every address";
|
|
ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11);
|
|
}
|
|
{
|
|
# The routing half: the SNI picks the backend and the only name that
|
|
# resolves to one is the store's own. A `default` that pointed anywhere
|
|
# would make this host a relay for whatever name a client invented.
|
|
name = "the passthrough routes only the store's name, to its loopback listener";
|
|
ok =
|
|
let
|
|
s = baoStream baoPkcs11;
|
|
in
|
|
lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s
|
|
&& lib.hasInfix "bao.t.local 127.0.0.1:8200;" s
|
|
&& lib.hasInfix ''default "";'' s;
|
|
}
|
|
{
|
|
# The listener is only half of reachable: the bridge firewall drops
|
|
# everything not named here, and a silent drop is the failure that reads
|
|
# as "the store is down" from inside a container.
|
|
name = "the store's port is open on the bridge where the store runs";
|
|
ok = builtins.elem 8200 (bridgePorts baoPkcs11);
|
|
}
|
|
{
|
|
# Raft refuses to start without it, and says so in a message that names
|
|
# neither the setting nor the stanza.
|
|
name = "the store advertises a cluster address";
|
|
ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or "");
|
|
}
|
|
{
|
|
# The listener trusts the agent CA through a file no cert-auth role
|
|
# names. At least one listener verifies clients, so an empty set cannot
|
|
# pass.
|
|
name = "every client-verifying listener reads the listener-only bundle";
|
|
ok =
|
|
let
|
|
verifying = lib.filter (l: l ? tls_client_ca_file) (
|
|
lib.attrValues (baoSettings baoPkcs11).listener
|
|
);
|
|
in
|
|
verifying != [ ]
|
|
&& lib.all (l: l.tls_client_ca_file == "${tlsDir}/listener-client-ca.pem") verifying;
|
|
}
|
|
{
|
|
# The other half of keeping agents out of host roles: those roles pin
|
|
# the store CA alone. The positive count is the control that the text
|
|
# searched is the one the roles are written in.
|
|
name = "host cert-auth roles pin client-ca.pem and never the listener bundle or the agent CA";
|
|
ok =
|
|
let
|
|
scripts = lib.concatStrings (
|
|
lib.mapAttrsToList (
|
|
n: u: lib.optionalString (lib.hasPrefix "swarm-bao-" n) (u.script or "")
|
|
) baoPkcs11.systemd.services
|
|
);
|
|
in
|
|
lib.hasInfix "certificate=@${tlsDir}/client-ca.pem" scripts
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/listener-client-ca.pem" scripts)
|
|
&& !(lib.hasInfix "certificate=@${tlsDir}/agent-ca.pem" scripts);
|
|
}
|
|
{
|
|
# Both writers compose the bundle through one function, which refuses
|
|
# a result that does not begin with the store CA and replaces the file
|
|
# only when its bytes change.
|
|
name = "both bundle writers use the one composer, which keeps the store CA first";
|
|
ok =
|
|
let
|
|
s = baoPkcs11.systemd.services;
|
|
composes =
|
|
u:
|
|
lib.hasInfix "compose_listener_bundle() {" u.script
|
|
&& lib.hasInfix "\ncompose_listener_bundle\n" u.script
|
|
&& lib.hasInfix ''cmp -s -n "$(stat -c %s ${tlsDir}/client-ca.pem)" ${tlsDir}/client-ca.pem "$tmp"'' u.script
|
|
&& lib.hasInfix ''cmp -s "$tmp" ${tlsDir}/listener-client-ca.pem'' u.script
|
|
&& lib.hasInfix "mktemp -p ${tlsDir} " u.script;
|
|
in
|
|
s ? swarm-bao-agent-pki && composes s.swarm-bao-certs && composes s.swarm-bao-agent-pki;
|
|
}
|
|
{
|
|
# openbao reads its client-CA file only at start, so picking up the
|
|
# agent CA is a restart: automatic where the store unseals itself,
|
|
# printed where a human has to. The shamir arm is the control.
|
|
name = "the agent PKI unit restarts openbao under pkcs11 and only prints the step under shamir";
|
|
ok =
|
|
let
|
|
restart = ''systemctl --machine="$machine" restart openbao.service'';
|
|
p = baoPkcs11.systemd.services.swarm-bao-agent-pki.script;
|
|
sh = baoShamir.systemd.services.swarm-bao-agent-pki.script;
|
|
in
|
|
lib.hasInfix restart p
|
|
&& lib.hasInfix ''if [ "$written_us" -le "$started_us" ]; then'' p
|
|
&& !(lib.hasInfix restart sh)
|
|
&& lib.hasInfix "# then unseal" sh;
|
|
}
|
|
{
|
|
# The browser UI's listener must not loosen the one every reader dials.
|
|
# `loopback` is named so an empty filter cannot pass.
|
|
name = "every listener but metrics and the UI still requires a client certificate";
|
|
ok =
|
|
let
|
|
l = (baoSettings baoPkcs11).listener;
|
|
api = builtins.removeAttrs l [
|
|
"metrics"
|
|
"ui"
|
|
];
|
|
in
|
|
api ? loopback
|
|
&& lib.all (x: (x.tls_require_and_verify_client_cert or false) == true) (lib.attrValues api);
|
|
}
|
|
{
|
|
# No client certificate on this one, so loopback is all that keeps it
|
|
# away from everything but this netns.
|
|
name = "the UI listener is loopback-only and the UI is served";
|
|
ok =
|
|
let
|
|
s = baoSettings baoPkcs11;
|
|
in
|
|
(s.ui or false) == true
|
|
&& lib.hasPrefix "127.0.0.1:" (s.listener.ui.address or "")
|
|
&& !(s.listener.ui ? tls_require_and_verify_client_cert);
|
|
}
|
|
{
|
|
# The nginx in front of it is the listener's only client and must bind
|
|
# loopback too, and it refuses the endpoints a browser never needs.
|
|
name = "the UI's nginx binds loopback and refuses the unseal and root-generation paths";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.containers.swarm-bao.config.services.nginx.virtualHosts.bao-ui;
|
|
in
|
|
lib.all (x: x.addr == "127.0.0.1") v.listen
|
|
&& (v.locations."~* ^/v1/sys/(unseal|seal|step-down)/?$".return or null) == "403"
|
|
&& (v.locations."~* ^/v1/sys/(rekey|generate-root)".return or null) == "403";
|
|
}
|
|
{
|
|
# A new vhost gets no group gate from `auth_request`; authelia's
|
|
# default policy admits any session. Evaluated on a host that runs
|
|
# authelia and no store, because that is where the rule has to render.
|
|
name = "authelia restricts the store's UI name to admins wherever authelia runs";
|
|
ok =
|
|
let
|
|
rules =
|
|
autheliaOnly.containers.swarm-authelia.config.services.authelia.instances.swarm.settings.access_control.rules;
|
|
in
|
|
!autheliaOnly.services.hyperhive.deploy.bao.enable
|
|
&& builtins.elem {
|
|
domain = "bao-ui.t.local";
|
|
subject = [ "group:admins" ];
|
|
policy = "one_factor";
|
|
} rules;
|
|
}
|
|
{
|
|
# The other half of that rule: the vhost exists, and asks authelia.
|
|
name = "the store's host serves the UI name behind the authelia subrequest";
|
|
ok =
|
|
let
|
|
v = baoPkcs11.services.nginx.virtualHosts."bao-ui.t.local";
|
|
in
|
|
lib.hasInfix "auth_request /__hive_authelia;" (v.locations."/".extraConfig or "")
|
|
&& (v.locations."/".proxyPass or "") == "http://127.0.0.1:8206";
|
|
}
|
|
{
|
|
# The stream passthrough every reader dials rides the same nginx as the
|
|
# UI's vhost. On a host whose only swarm name is the UI, that nginx must
|
|
# not wait on the services leaf, which needs a login to this very store.
|
|
# The control is a store host that also fronts forge: its gateway waits,
|
|
# exactly as it did before the UI existed.
|
|
name = "a store host fronting only the UI does not hold nginx on a store login";
|
|
ok =
|
|
let
|
|
waits =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.requiredBy;
|
|
orders =
|
|
m:
|
|
builtins.elem "hive-gateway-self-signed-cert.service" m.systemd.services.swarm-services-cert.before;
|
|
in
|
|
baoPkcs11.services.hyperhive.swarm.localServiceDomains == [ "bao-ui.t.local" ]
|
|
&& baoPkcs11.systemd.services ? hive-gateway-self-signed-cert
|
|
&& !(waits baoPkcs11)
|
|
&& !(orders baoPkcs11)
|
|
&& waits baoWithForge
|
|
&& orders baoWithForge;
|
|
}
|
|
];
|
|
in
|
|
runGroup "bao-basics" cases
|