hyperhive/nix/module-eval/bao-grants.nix
atlas 87174863ca swarm-bao: grant the controller read on the agent credential prefix
mint_and_verify reads the queue credential back before writing it, so a
re-run keeps the value a live agent already authenticates with. that read
is read_optional, which maps only a 404 to absence — so with create/update
alone every mint aborted on a 403 at its first store read.

read on the same paths the stanza already grants create and update, and
nothing else: no list, no delete, no patch.
2026-09-23 18:55:02 +02:00

552 lines
26 KiB
Nix

# `checks.module-eval-bao-grants` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
;
# The store, plus a placed bootstrap token: the only shape in which the
# swarm's first grant can be written at all.
baoGrantHere = hive {
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
};
# The credential without the store. Writing the first grant is a store-side
# operation, so a host holding only the token has nothing to do — and this
# is the arm that separates "an operator placed a token" from "this box can
# act on it".
baoGrantNoStore = hive {
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
};
# The store and the token, with no CA to trust. `mkForce` because the PKI
# glue supplies one by default here — this is the deployment that brings its
# own certificates and has not named the authority yet, and it separates
# "the grant unit runs" from "cert auth can be set up".
baoGrantNoClientCa = hive {
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
deploy.bao.clientCaFile = lib.mkForce null;
};
# The store plus every one of the four readers that used to log in as the
# hive. One fixture rather than four: the claim they are four *separate*
# principals is only testable where all four render at once — that is the
# deployment in which two of them sharing a leaf would be invisible.
baoGrantWithConsumers = hive {
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
deploy.matrix.enable = true;
deploy.grafana.enable = true;
deploy.swarm-otel.enable = true;
};
cases = [
{
# Reads the rendered unit on the HOST, which is where the write happens:
# every API listener demands a client certificate, and the host is the
# side that has one.
name = "a store host with a placed bootstrap token renders the granting unit on the host";
ok =
let
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
in
lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script
&& u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token";
}
{
# The move is the fix, so pin the side it landed on: in the container it
# had no identity to open a connection with, and no address that resolved
# to the store from its own netns.
name = "the granting unit is not rendered inside the store's container";
ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy);
}
{
# `StartLimit*` are `[Unit]` settings that systemd ignores under
# `[Service]`, so a bound written into `serviceConfig` renders, deploys
# and does nothing. Asserted where nixpkgs puts it rather than where it
# was written. The values are pinned because they are the bound: under
# `shamir` a human unseals by hand, and anything shorter than a day gives
# up first — `start-limit-hit` does not self-heal.
name = "the granting unit's start limit lands in [Unit], not [Service]";
ok =
let
u = baoGrantHere.systemd.services.swarm-bao-controller-policy;
in
toString u.unitConfig.StartLimitBurst == "2880"
&& toString u.unitConfig.StartLimitIntervalSec == "90000"
&& !(u.serviceConfig ? StartLimitBurst);
}
{
# The grants themselves, and the `hive-` prefix is the whole point:
# without it the controller can rewrite the policy that constrains it,
# which is a privilege escalation that renders, deploys and looks fine.
# Readable here only because the HCL is piped as an argument rather than
# written to a store path.
name = "the controller's bao grants cannot reach the policy that constrains it";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s);
}
{
# Same host-side reasoning as the controller's granting unit above: the
# write needs a client certificate and the host is the side that has one.
name = "a store host with a placed bootstrap token renders the publisher's granting unit too";
ok =
let
u = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy;
in
u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"
&& lib.hasInfix "swarm-secret-publisher" u.script;
}
{
# The control for the case above, and the same one the controller's unit
# has: rendered on the host means NOT rendered in the container, where it
# would have neither an identity nor a route to the store.
name = "the publisher's granting unit is not rendered inside the store's container";
ok =
!(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy);
}
{
# The whole point of a second principal. The two prefixes it publishes to
# and not `swarm/`, so it cannot touch an agent's credentials; and no
# `read`, so a unit whose job is copying a file cannot recover what is
# already there. Pinned as the full capability list per prefix, because an
# added capability is exactly what a presence check misses.
name = "the publisher's grant is write-only and reaches the hive and service prefixes alone";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s
&& lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/*" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# The ordering is load-bearing and invisible at runtime: the controller's
# unit creates the KV and cert-auth mounts this one writes into, so
# without it a cold boot races and fails with "route entry not found",
# which names neither unit.
name = "the publisher's granting unit is ordered after the one that creates the mounts";
ok = lib.elem "swarm-bao-controller-policy.service" (
baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.after
);
}
{
# The third principal's grant, and the narrowest of the three: ONE path,
# spelled to the leaf. The negative arms are the property — a homeserver
# is not entitled to overwrite Grafana's OIDC client, so widening this to
# the `services/` prefix the publisher holds would be a real loss even
# though it would read as tidier.
#
# ⚠️ `hives` is PLURAL, because the path segment comes from
# `Kind::Hive`'s strum serialisation and not from `Kind::label`, which
# renders the singular for error text. The singular spelling evaluates,
# deploys, and 403s every read with "permission denied" and nothing else.
#
# 🩸 The hive NAME in the middle is the per-hive half of this credential:
# the token used to be one swarm-wide value under `services/matrix/`,
# which every hive's own policy granted read on. The negative arms below
# are what keep it from drifting back — neither the `services/*` tree nor
# a `hives/*` wildcard may appear, since either one hands matrix-ctl (or
# a hive) reach beyond the single leaf it owns.
name = "matrix-ctl's grant is one hive's sender token path and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/h1/matrix/sender-token\" {" s
&& !(lib.hasInfix "secret/data/swarm/services" s)
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# 🩸 `read` is load-bearing here, and the publisher — the one sibling
# that still has no `read` — shows what its absence costs. matrix-ctl's
# first act is to read this path back and stop if something is there —
# that read IS "and only once", so without the capability every container
# restart would mint a second access token and invalidate the hive's.
# (The controller holds `read` for the same idempotency reason, on the
# agent prefix.)
name = "matrix-ctl may read back the one path it writes";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.script;
in
lib.hasInfix "capabilities = [\"create\", \"update\", \"read\"]" s
&& lib.hasInfix "auth/cert/certs/swarm-matrix-ctl" s
&& lib.hasInfix "allowed_common_names=swarm-matrix-ctl" s;
}
{
# Same two controls its siblings carry: ordered after the unit that makes
# the mounts it writes into, and rendered on the HOST rather than inside
# the store's container, where it would have neither an identity nor a
# route to the store.
name = "matrix-ctl's granting unit is ordered after the mounts and rendered on the host";
ok =
lib.elem "swarm-bao-controller-policy.service" (
baoGrantHere.systemd.services.swarm-bao-matrix-ctl-policy.after
)
&& !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-matrix-ctl-policy);
}
# ── the four readers that used to share the hive's own leaf ──────────────
#
# 🩸 Until this split all four presented `deploy.bao.clientCertFile`, whose
# policy grants read on `swarm/agents/*`, `swarm/hives/<hive>/*` AND
# `swarm/services/*`. Four principals behind one certificate are one
# principal to bao, so the only expressible grant was the union: the unit
# fetching Grafana's OIDC secret could fetch every agent credential in the
# swarm.
#
# Every one of these cases carries the same three negative arms, and they
# are the deliverable rather than decoration — a positive arm alone passes
# just as well when the other two stanzas are still there beside it. The
# arms pin what each principal must NOT reach, so a later widening fails
# here instead of being noticed in a store.
{
name = "the matrix-token reader's grant is one hive's appservice token and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-token-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/h1/matrix/appservice-token\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
# The three stanzas the hive's own leaf carried, none of which this
# principal needs: every agent's credential, every service's OIDC
# client, and the rest of its own hive's tree — including the queue
# credential its sibling reader fetches.
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/services" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/*" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/queue" s)
# A `hives/*` wildcard would serve every hive from one role and let any
# hive read any other's token — the reach this split exists to remove,
# not to create.
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
# Nothing may rewrite the policy constraining it, for the reason the
# controller's own `hive-*` narrowing above gives.
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
name = "the queue-credential reader's grant is one hive's queue credential and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-queue-agent-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/h1/queue/agent\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/services" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/*" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/matrix" s)
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# ⚠️ The client id is the path segment, so the negative arm that matters
# for this one is the OTHER service's: `services/*` would have granted
# both, and the two are separate principals precisely because a
# dashboard is not entitled to a collector's credential.
name = "the Grafana OIDC reader's grant is Grafana's own client secret and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/swarm-grafana/oidc/client\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-collector" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# The mirror of the case above, and the arm naming `swarm-grafana` is why
# these are two principals rather than one `services/*` grant shared.
name = "the collector OIDC reader's grant is the collector's own client secret and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-otel-oidc-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/swarm-collector/oidc/client\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-grafana" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# 🩸 The half that makes the policies above bind: a policy grants only
# through a token that carries it, and a token is minted by a cert-auth
# role matching a CN. Four distinct subjects is the whole mechanism — one
# subject for four readers is one principal however the policies read.
#
# The per-hive subjects carry the hive name because their paths do; the
# two service subjects do not, because an OIDC client is registered once
# per swarm. Pinned so neither shape is tidied into the other.
name = "each of the four readers logs in under a subject of its own";
ok =
let
subjectOf =
unit: role: cn:
let
s = baoGrantHere.systemd.services.${unit}.script;
in
lib.hasInfix "auth/cert/certs/${role}" s
&& lib.hasInfix "allowed_common_names=${cn}" s
&& lib.hasInfix "token_policies=${role}" s
# Outside the `hive-*` namespace the controller may rewrite, for
# the reason the three service principals above state.
&& !(lib.hasInfix "auth/cert/certs/hive-" s);
in
subjectOf "swarm-bao-matrix-token-policy" "swarm-matrix-token-h1" "swarm-bao-matrix-token-h1"
&& subjectOf "swarm-bao-queue-agent-policy" "swarm-queue-agent-h1" "swarm-bao-queue-agent-h1"
&& subjectOf "swarm-bao-grafana-oidc-policy" "swarm-grafana-oidc" "swarm-bao-grafana-oidc"
&& subjectOf "swarm-bao-otel-oidc-policy" "swarm-otel-oidc" "swarm-bao-otel-oidc";
}
{
# 🩸 The consuming side, and the arm that would catch the regression that
# costs the most: a unit repointed back at `deploy.bao.clientCertFile`
# evaluates, deploys and logs in — and silently restores the union grant,
# because bao would again see one principal. Nothing about the policies
# above would look wrong.
#
# Each pair is asserted whole: a certificate with no key authenticates
# nothing, so a half-set pair is a reader that does not render.
name = "each of the four readers presents its own leaf, never the hive's";
ok =
let
b = baoGrantWithConsumers.services.hyperhive.deploy.bao;
hiveLeaf = [
b.clientCertFile
b.clientKeyFile
];
own = [
b.matrixTokenClientCertFile
b.matrixTokenClientKeyFile
b.queueAgentClientCertFile
b.queueAgentClientKeyFile
b.grafanaOidcClientCertFile
b.grafanaOidcClientKeyFile
b.otelOidcClientCertFile
b.otelOidcClientKeyFile
];
envOf = unit: baoGrantWithConsumers.systemd.services.${unit}.environment;
presents =
unit: cert: key:
(envOf unit).BAO_CLIENT_CERT == cert && (envOf unit).BAO_CLIENT_KEY == key;
in
lib.all (p: p != null) own
&& !(lib.any (p: lib.elem p hiveLeaf) own)
&& lib.length (lib.unique own) == lib.length own
&& presents "swarm-bao-matrix-token" b.matrixTokenClientCertFile b.matrixTokenClientKeyFile
&& presents "swarm-bao-queue-agent" b.queueAgentClientCertFile b.queueAgentClientKeyFile
&& presents "swarm-bao-grafana-oidc" b.grafanaOidcClientCertFile b.grafanaOidcClientKeyFile
&& presents "swarm-bao-otel-oidc" b.otelOidcClientCertFile b.otelOidcClientKeyFile;
}
{
# The minting side of the same claim. A role matching a subject nothing
# signs is a reader that cannot log in, so the leaves and the roles have
# to be asserted against each other — and the two per-hive leaves carry
# THIS host's hive name, which is what makes one hive's leaf useless
# against another hive's role.
name = "the PKI unit signs a leaf per reader, each under that reader's own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
# The basename and the subject are matched separately: `signLeaf` takes
# them as consecutive arguments across a `\` continuation, so one
# literal spanning both would pin this file's line wrapping rather than
# the pairing it means to.
lib.all (lib.flip lib.hasInfix s) [
"/matrix-token.pem ]"
"swarm-bao-matrix-token-h1 \"\" clientAuth"
"/queue-agent.pem ]"
"swarm-bao-queue-agent-h1 \"\" clientAuth"
"/grafana-oidc.pem ]"
"swarm-bao-grafana-oidc \"\" clientAuth"
"/otel-oidc.pem ]"
"swarm-bao-otel-oidc \"\" clientAuth"
];
}
{
# The absence arm: with no client CA there is no trust anchor, so the
# login roles cannot be written — but the policies they would attach are
# still asserted, exactly as the three service principals above behave in
# this deployment. A unit that vanished here would take the policy with
# it and leave nothing to diagnose.
name = "with no client CA the four readers get policies but no login roles";
ok =
let
units = [
"swarm-bao-matrix-token-policy"
"swarm-bao-queue-agent-policy"
"swarm-bao-grafana-oidc-policy"
"swarm-bao-otel-oidc-policy"
];
scriptOf = unit: baoGrantNoClientCa.systemd.services.${unit}.script;
in
lib.all (
unit:
(baoGrantNoClientCa.systemd.services ? ${unit})
&& lib.hasInfix "bao policy write" (scriptOf unit)
&& !(lib.hasInfix "auth/cert/certs" (scriptOf unit))
) units;
}
{
# Same control the three service principals carry: the write needs a
# client certificate and the host is the side that has one, so a unit
# rendered inside the store's container would have neither an identity
# nor a route. Plus the ordering that makes the mounts exist first.
name = "the four readers' granting units are ordered after the mounts and rendered on the host";
ok =
let
units = [
"swarm-bao-matrix-token-policy"
"swarm-bao-queue-agent-policy"
"swarm-bao-grafana-oidc-policy"
"swarm-bao-otel-oidc-policy"
];
in
lib.all (
unit:
lib.elem "swarm-bao-controller-policy.service" baoGrantHere.systemd.services.${unit}.after
&& !(baoGrantHere.containers.swarm-bao.config.systemd.services ? ${unit})
) units;
}
{
# A store host that has not placed a bootstrap token can write no grant at
# all, so none of the four units may exist — the same claim
# `baoGrantNoStore` makes for the controller's, one file over. Without
# this arm `lib.mkIf haveBootstrapToken` could be dropped from the shared
# builder and every other case here would still pass.
name = "without a bootstrap token none of the four readers' granting units render";
ok =
let
s = baoGrantNoStore.systemd.services;
in
!(s ? swarm-bao-matrix-token-policy)
&& !(s ? swarm-bao-queue-agent-policy)
&& !(s ? swarm-bao-grafana-oidc-policy)
&& !(s ? swarm-bao-otel-oidc-policy);
}
{
# The policy authorising this route lives in another file, and nothing
# else relates the grants to the paths the code actually writes.
#
# `secret/data/` is KV v2's ACL prefix; `swarm` is
# `swarm_secret_client::path::ROOT` and `agents` is
# `Kind::Agent.as_str()`, both of which that crate pins in its own test.
#
# The grant is still the agent kind alone because nothing writes another
# one yet. It widens when a path outside `agents/` gains a writer, not
# when the kinds are declared.
name = "the controller may write agent credentials, and only under the agent prefix";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "secret/data/swarm/agents/*" s
&& !(lib.hasInfix "secret/data/*" s)
&& !(lib.hasInfix "path \"secret/*\"" s);
}
{
# The exact list is the property, not an accident of how it was typed.
# `read` is in it because `mint_and_verify` reads a queue credential back
# before rewriting it; `list` is not, so the controller can fetch a
# credential only for an agent it was handed the name of, never enumerate
# the tree. Pinned as the whole capability list, because an added
# capability is exactly what a presence check misses.
name = "the controller's grant on agent credentials is create/read/update and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"read\", \"update\"]" s;
}
{
# The policy above grants paths under a mount nothing else creates, so
# the unit that writes the policy has to create it too — otherwise every
# certificate login fails against a path that is not there.
name = "the granting unit creates the cert auth mount and the controller's role";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "bao auth enable cert" s
&& lib.hasInfix "auth/cert/certs/swarm-controller" s
&& lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s;
}
{
# Same shape as the cert mount above, for the engine the controller
# writes credentials through: a fresh store has no `secret/`, so the
# grant would name a mount nobody created and the first write would 404.
#
# ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below
# spells out: the policy text is embedded in this same script and grants
# `secret/data/...`, so any arm keyed on the *path* is satisfied either
# way and could never fail.
name = "the granting unit creates the KV mount the controller writes through";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "bao secrets enable -path=secret kv-v2" s;
}
{
# The arm that makes the one above mean something. A role's trust anchor
# is the CA, so with none named there is nothing to write — and the
# policy write, which needs no CA, must survive that.
#
# ⚠️ Matched on the COMMANDS, not on `auth/cert/certs`: the policy text is
# embedded in this same script and grants that very path, so the shorter
# infix is present either way and the arm could never fail.
name = "with no client CA the unit still writes the policy and skips the role";
ok =
let
s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script;
in
lib.hasInfix "bao policy write" s
&& !(lib.hasInfix "bao auth enable cert" s)
&& !(lib.hasInfix "client-ca.pem" s)
# The KV mount is NOT part of what a missing client CA switches off:
# the controller writes through it whether or not anything can log in
# by certificate. Asserted here rather than trusted, because both
# steps live in the same script and one indentation level decides it.
&& lib.hasInfix "bao secrets enable -path=secret kv-v2" s;
}
{
# What makes the granting-unit cases mean something, and the property
# the host-side half depends on: no store here, so no bind mount and no
# unit. Without it a hive that merely names a token would drag the
# store's container config into its evaluation.
name = "a bootstrap token on a host that runs no store grants nothing";
ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir);
}
];
in
runGroup "bao-grants" cases