# `checks.module-eval-bao-grants` — see ./lib.nix for the shared # rationale (why this suite exists, naming convention, "evaluates # not executes"). { pkgs, lib, self, nixosSystem, }: let inherit (import ./lib.nix { inherit pkgs lib self nixosSystem ; }) hive runGroup ; # The store, plus a placed bootstrap token: the only shape in which the # swarm's first grant can be written at all. baoGrantHere = hive { deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; }; # The credential without the store. Writing the first grant is a store-side # operation, so a host holding only the token has nothing to do — and this # is the arm that separates "an operator placed a token" from "this box can # act on it". baoGrantNoStore = hive { deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; }; # The store and the token, with no CA to trust. `mkForce` because the PKI # glue supplies one by default here — this is the deployment that brings its # own certificates and has not named the authority yet, and it separates # "the grant unit runs" from "cert auth can be set up". baoGrantNoClientCa = hive { deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; deploy.bao.clientCaFile = lib.mkForce null; }; # The store plus every one of the four readers that used to log in as the # hive. One fixture rather than four: the claim they are four *separate* # principals is only testable where all four render at once — that is the # deployment in which two of them sharing a leaf would be invisible. baoGrantWithConsumers = hive { deploy.bao.enable = true; deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; deploy.matrix.enable = true; deploy.grafana.enable = true; deploy.swarm-otel.enable = true; }; cases = [ { # Reads the rendered unit on the HOST, which is where the write happens: # every API listener demands a client certificate, and the host is the # side that has one. name = "a store host with a placed bootstrap token renders the granting unit on the host"; ok = let u = baoGrantHere.systemd.services.swarm-bao-controller-policy; in lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; } { # The move is the fix, so pin the side it landed on: in the container it # had no identity to open a connection with, and no address that resolved # to the store from its own netns. name = "the granting unit is not rendered inside the store's container"; ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy); } { # `StartLimit*` are `[Unit]` settings that systemd ignores under # `[Service]`, so a bound written into `serviceConfig` renders, deploys # and does nothing. Asserted where nixpkgs puts it rather than where it # was written. The values are pinned because they are the bound: under # `shamir` a human unseals by hand, and anything shorter than a day gives # up first — `start-limit-hit` does not self-heal. name = "the granting unit's start limit lands in [Unit], not [Service]"; ok = let u = baoGrantHere.systemd.services.swarm-bao-controller-policy; in toString u.unitConfig.StartLimitBurst == "2880" && toString u.unitConfig.StartLimitIntervalSec == "90000" && !(u.serviceConfig ? StartLimitBurst); } { # The grants themselves, and the `hive-` prefix is the whole point: # without it the controller can rewrite the policy that constrains it, # which is a privilege escalation that renders, deploys and looks fine. # Readable here only because the HCL is piped as an argument rather than # written to a store path. name = "the controller's bao grants cannot reach the policy that constrains it"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); } { # Same host-side reasoning as the controller's granting unit above: the # write needs a client certificate and the host is the side that has one. name = "a store host with a placed bootstrap token renders the publisher's granting unit too"; ok = let u = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy; in u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token" && lib.hasInfix "swarm-secret-publisher" u.script; } { # The control for the case above, and the same one the controller's unit # has: rendered on the host means NOT rendered in the container, where it # would have neither an identity nor a route to the store. name = "the publisher's granting unit is not rendered inside the store's container"; ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); } { # The whole point of a second principal. The two prefixes it publishes to # and not `swarm/`, so it cannot touch an agent's credentials; and no # `read`, so a unit whose job is copying a file cannot recover what is # already there. Pinned as the full capability list per prefix, because an # added capability is exactly what a presence check misses. name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; ok = let s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/*" s) && !(lib.hasInfix "sys/policies/acl" s); } { # The ordering is load-bearing and invisible at runtime: the controller's # unit creates the KV and cert-auth mounts this one writes into, so # without it a cold boot races and fails with "route entry not found", # which names neither unit. name = "the publisher's granting unit is ordered after the one that creates the mounts"; ok = lib.elem "swarm-bao-controller-policy.service" ( baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.after ); } { # The third principal's grant, and the narrowest of the three: ONE path, # spelled to the leaf. The negative arms are the property — a homeserver # is not entitled to overwrite Grafana's OIDC client, so widening this to # the `services/` prefix the publisher holds would be a real loss even # though it would read as tidier. # # ⚠️ `hives` is PLURAL, because the path segment comes from # `Kind::Hive`'s strum serialisation and not from `Kind::label`, which # renders the singular for error text. The singular spelling evaluates, # deploys, and 403s every read with "permission denied" and nothing else. # # 🩸 The hive NAME in the middle is the per-hive half of this credential: # the token used to be one swarm-wide value under `services/matrix/`, # which every hive's own policy granted read on. The negative arms below # are what keep it from drifting back — neither the `services/*` tree nor # a `hives/*` wildcard may appear, since either one hands matrix-ctl (or # a hive) reach beyond the single leaf it owns. name = "matrix-ctl's grant is one hive's sender token path and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/h1/matrix/sender-token\" {" s && !(lib.hasInfix "secret/data/swarm/services" s) && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/hives/*" s) && !(lib.hasInfix "sys/policies/acl" s); } { # 🩸 `read` is load-bearing here, and the publisher — the one sibling # that still has no `read` — shows what its absence costs. matrix-ctl's # first act is to read this path back and stop if something is there — # that read IS "and only once", so without the capability every container # restart would mint a second access token and invalidate the hive's. # (The controller holds `read` for the same idempotency reason, on the # agent prefix.) name = "matrix-ctl may read back the one path it writes"; ok = let s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script; in lib.hasInfix "capabilities = [\"create\", \"update\", \"read\"]" s && lib.hasInfix "auth/cert/certs/swarm-matrix-ctl" s && lib.hasInfix "allowed_common_names=swarm-matrix-ctl" s; } { # Same two controls its siblings carry: ordered after the unit that makes # the mounts it writes into, and rendered on the HOST rather than inside # the store's container, where it would have neither an identity nor a # route to the store. name = "matrix-ctl's granting unit is ordered after the mounts and rendered on the host"; ok = lib.elem "swarm-bao-controller-policy.service" ( baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.after ) && !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-matrix-ctl-policy); } # ── the four readers that used to share the hive's own leaf ────────────── # # 🩸 Until this split all four presented `deploy.bao.clientCertFile`, whose # policy grants read on `swarm/agents/*`, `swarm/hives//*` AND # `swarm/services/*`. Four principals behind one certificate are one # principal to bao, so the only expressible grant was the union: the unit # fetching Grafana's OIDC secret could fetch every agent credential in the # swarm. # # Every one of these cases carries the same three negative arms, and they # are the deliverable rather than decoration — a positive arm alone passes # just as well when the other two stanzas are still there beside it. The # arms pin what each principal must NOT reach, so a later widening fails # here instead of being noticed in a store. { name = "the matrix-token reader's grant is one hive's appservice token and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-matrix-token-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/h1/matrix/appservice-token\" {" s && lib.hasInfix "capabilities = [\"read\"]" s # The three stanzas the hive's own leaf carried, none of which this # principal needs: every agent's credential, every service's OIDC # client, and the rest of its own hive's tree — including the queue # credential its sibling reader fetches. && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/services" s) && !(lib.hasInfix "secret/data/swarm/hives/h1/*" s) && !(lib.hasInfix "secret/data/swarm/hives/h1/queue" s) # A `hives/*` wildcard would serve every hive from one role and let any # hive read any other's token — the reach this split exists to remove, # not to create. && !(lib.hasInfix "secret/data/swarm/hives/*" s) # Nothing may rewrite the policy constraining it, for the reason the # controller's own `hive-*` narrowing above gives. && !(lib.hasInfix "sys/policies/acl" s); } { name = "the queue-credential reader's grant is one hive's queue credential and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-queue-agent-policy.script; in lib.hasInfix "path \"secret/data/swarm/hives/h1/queue/agent\" {" s && lib.hasInfix "capabilities = [\"read\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/services" s) && !(lib.hasInfix "secret/data/swarm/hives/h1/*" s) && !(lib.hasInfix "secret/data/swarm/hives/h1/matrix" s) && !(lib.hasInfix "secret/data/swarm/hives/*" s) && !(lib.hasInfix "sys/policies/acl" s); } { # ⚠️ The client id is the path segment, so the negative arm that matters # for this one is the OTHER service's: `services/*` would have granted # both, and the two are separate principals precisely because a # dashboard is not entitled to a collector's credential. name = "the Grafana OIDC reader's grant is Grafana's own client secret and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script; in lib.hasInfix "path \"secret/data/swarm/services/swarm-grafana/oidc/client\" {" s && lib.hasInfix "capabilities = [\"read\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/hives" s) && !(lib.hasInfix "secret/data/swarm/services/*" s) && !(lib.hasInfix "swarm-collector" s) && !(lib.hasInfix "sys/policies/acl" s); } { # The mirror of the case above, and the arm naming `swarm-grafana` is why # these are two principals rather than one `services/*` grant shared. name = "the collector OIDC reader's grant is the collector's own client secret and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-otel-oidc-policy.script; in lib.hasInfix "path \"secret/data/swarm/services/swarm-collector/oidc/client\" {" s && lib.hasInfix "capabilities = [\"read\"]" s && !(lib.hasInfix "secret/data/swarm/agents" s) && !(lib.hasInfix "secret/data/swarm/hives" s) && !(lib.hasInfix "secret/data/swarm/services/*" s) && !(lib.hasInfix "swarm-grafana" s) && !(lib.hasInfix "sys/policies/acl" s); } { # 🩸 The half that makes the policies above bind: a policy grants only # through a token that carries it, and a token is minted by a cert-auth # role matching a CN. Four distinct subjects is the whole mechanism — one # subject for four readers is one principal however the policies read. # # The per-hive subjects carry the hive name because their paths do; the # two service subjects do not, because an OIDC client is registered once # per swarm. Pinned so neither shape is tidied into the other. name = "each of the four readers logs in under a subject of its own"; ok = let subjectOf = unit: role: cn: let s = baoGrantHere.systemd.services.${unit}.script; in lib.hasInfix "auth/cert/certs/${role}" s && lib.hasInfix "allowed_common_names=${cn}" s && lib.hasInfix "token_policies=${role}" s # Outside the `hive-*` namespace the controller may rewrite, for # the reason the three service principals above state. && !(lib.hasInfix "auth/cert/certs/hive-" s); in subjectOf "swarm-bao-matrix-token-policy" "swarm-matrix-token-h1" "swarm-bao-matrix-token-h1" && subjectOf "swarm-bao-queue-agent-policy" "swarm-queue-agent-h1" "swarm-bao-queue-agent-h1" && subjectOf "swarm-bao-grafana-oidc-policy" "swarm-grafana-oidc" "swarm-bao-grafana-oidc" && subjectOf "swarm-bao-otel-oidc-policy" "swarm-otel-oidc" "swarm-bao-otel-oidc"; } { # 🩸 The consuming side, and the arm that would catch the regression that # costs the most: a unit repointed back at `deploy.bao.clientCertFile` # evaluates, deploys and logs in — and silently restores the union grant, # because bao would again see one principal. Nothing about the policies # above would look wrong. # # Each pair is asserted whole: a certificate with no key authenticates # nothing, so a half-set pair is a reader that does not render. name = "each of the four readers presents its own leaf, never the hive's"; ok = let b = baoGrantWithConsumers.services.hyperhive.deploy.bao; hiveLeaf = [ b.clientCertFile b.clientKeyFile ]; own = [ b.matrixTokenClientCertFile b.matrixTokenClientKeyFile b.queueAgentClientCertFile b.queueAgentClientKeyFile b.grafanaOidcClientCertFile b.grafanaOidcClientKeyFile b.otelOidcClientCertFile b.otelOidcClientKeyFile ]; envOf = unit: baoGrantWithConsumers.systemd.services.${unit}.environment; presents = unit: cert: key: (envOf unit).BAO_CLIENT_CERT == cert && (envOf unit).BAO_CLIENT_KEY == key; in lib.all (p: p != null) own && !(lib.any (p: lib.elem p hiveLeaf) own) && lib.length (lib.unique own) == lib.length own && presents "swarm-bao-matrix-token" b.matrixTokenClientCertFile b.matrixTokenClientKeyFile && presents "swarm-bao-queue-agent" b.queueAgentClientCertFile b.queueAgentClientKeyFile && presents "swarm-bao-grafana-oidc" b.grafanaOidcClientCertFile b.grafanaOidcClientKeyFile && presents "swarm-bao-otel-oidc" b.otelOidcClientCertFile b.otelOidcClientKeyFile; } { # The minting side of the same claim. A role matching a subject nothing # signs is a reader that cannot log in, so the leaves and the roles have # to be asserted against each other — and the two per-hive leaves carry # THIS host's hive name, which is what makes one hive's leaf useless # against another hive's role. name = "the PKI unit signs a leaf per reader, each under that reader's own subject"; ok = let s = baoGrantHere.systemd.services.swarm-bao-pki.script; in # The basename and the subject are matched separately: `signLeaf` takes # them as consecutive arguments across a `\` continuation, so one # literal spanning both would pin this file's line wrapping rather than # the pairing it means to. lib.all (lib.flip lib.hasInfix s) [ "/matrix-token.pem ]" "swarm-bao-matrix-token-h1 \"\" clientAuth" "/queue-agent.pem ]" "swarm-bao-queue-agent-h1 \"\" clientAuth" "/grafana-oidc.pem ]" "swarm-bao-grafana-oidc \"\" clientAuth" "/otel-oidc.pem ]" "swarm-bao-otel-oidc \"\" clientAuth" ]; } { # The absence arm: with no client CA there is no trust anchor, so the # login roles cannot be written — but the policies they would attach are # still asserted, exactly as the three service principals above behave in # this deployment. A unit that vanished here would take the policy with # it and leave nothing to diagnose. name = "with no client CA the four readers get policies but no login roles"; ok = let units = [ "swarm-bao-matrix-token-policy" "swarm-bao-queue-agent-policy" "swarm-bao-grafana-oidc-policy" "swarm-bao-otel-oidc-policy" ]; scriptOf = unit: baoGrantNoClientCa.systemd.services.${unit}.script; in lib.all ( unit: (baoGrantNoClientCa.systemd.services ? ${unit}) && lib.hasInfix "bao policy write" (scriptOf unit) && !(lib.hasInfix "auth/cert/certs" (scriptOf unit)) ) units; } { # Same control the three service principals carry: the write needs a # client certificate and the host is the side that has one, so a unit # rendered inside the store's container would have neither an identity # nor a route. Plus the ordering that makes the mounts exist first. name = "the four readers' granting units are ordered after the mounts and rendered on the host"; ok = let units = [ "swarm-bao-matrix-token-policy" "swarm-bao-queue-agent-policy" "swarm-bao-grafana-oidc-policy" "swarm-bao-otel-oidc-policy" ]; in lib.all ( unit: lib.elem "swarm-bao-controller-policy.service" baoGrantHere.systemd.services.${unit}.after && !(baoGrantHere.containers.swarm-bao.config.systemd.services ? ${unit}) ) units; } { # A store host that has not placed a bootstrap token can write no grant at # all, so none of the four units may exist — the same claim # `baoGrantNoStore` makes for the controller's, one file over. Without # this arm `lib.mkIf haveBootstrapToken` could be dropped from the shared # builder and every other case here would still pass. name = "without a bootstrap token none of the four readers' granting units render"; ok = let s = baoGrantNoStore.systemd.services; in !(s ? swarm-bao-matrix-token-policy) && !(s ? swarm-bao-queue-agent-policy) && !(s ? swarm-bao-grafana-oidc-policy) && !(s ? swarm-bao-otel-oidc-policy); } { # The policy authorising this route lives in another file, and nothing # else relates the grants to the paths the code actually writes. # # `secret/data/` is KV v2's ACL prefix; `swarm` is # `swarm_secret_client::path::ROOT` and `agents` is # `Kind::Agent.as_str()`, both of which that crate pins in its own test. # # The grant is still the agent kind alone because nothing writes another # one yet. It widens when a path outside `agents/` gains a writer, not # when the kinds are declared. name = "the controller may write agent credentials, and only under the agent prefix"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "secret/data/swarm/agents/*" s && !(lib.hasInfix "secret/data/*" s) && !(lib.hasInfix "path \"secret/*\"" s); } { # The exact list is the property, not an accident of how it was typed. # `read` is in it because `mint_and_verify` reads a queue credential back # before rewriting it; `list` is not, so the controller can fetch a # credential only for an agent it was handed the name of, never enumerate # the tree. Pinned as the whole capability list, because an added # capability is exactly what a presence check misses. name = "the controller's grant on agent credentials is create/read/update and nothing else"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"read\", \"update\"]" s; } { # The policy above grants paths under a mount nothing else creates, so # the unit that writes the policy has to create it too — otherwise every # certificate login fails against a path that is not there. name = "the granting unit creates the cert auth mount and the controller's role"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao auth enable cert" s && lib.hasInfix "auth/cert/certs/swarm-controller" s && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; } { # Same shape as the cert mount above, for the engine the controller # writes credentials through: a fresh store has no `secret/`, so the # grant would name a mount nobody created and the first write would 404. # # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below # spells out: the policy text is embedded in this same script and grants # `secret/data/...`, so any arm keyed on the *path* is satisfied either # way and could never fail. name = "the granting unit creates the KV mount the controller writes through"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao secrets enable -path=secret kv-v2" s; } { # The arm that makes the one above mean something. A role's trust anchor # is the CA, so with none named there is nothing to write — and the # policy write, which needs no CA, must survive that. # # ⚠️ Matched on the COMMANDS, not on `auth/cert/certs`: the policy text is # embedded in this same script and grants that very path, so the shorter # infix is present either way and the arm could never fail. name = "with no client CA the unit still writes the policy and skips the role"; ok = let s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; in lib.hasInfix "bao policy write" s && !(lib.hasInfix "bao auth enable cert" s) && !(lib.hasInfix "client-ca.pem" s) # The KV mount is NOT part of what a missing client CA switches off: # the controller writes through it whether or not anything can log in # by certificate. Asserted here rather than trusted, because both # steps live in the same script and one indentation level decides it. && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; } { # What makes the granting-unit cases mean something, and the property # the host-side half depends on: no store here, so no bind mount and no # unit. Without it a hive that merely names a token would drag the # store's container config into its evaluation. name = "a bootstrap token on a host that runs no store grants nothing"; ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); } ]; in runGroup "bao-grants" cases