swarm-bao: give each hive-cert consumer its own bao identity

Four units read one path each out of the store, and all four logged in
holding `deploy.bao.clientCertFile` — the hive's own leaf. Bao identifies
a principal by the subject of the certificate it presents, so four
readers behind one certificate were ONE principal, and the only grant
expressible was the union of what the four need: read on
`swarm/agents/*`, `swarm/hives/<hive>/*` and `swarm/services/*`. The unit
fetching Grafana's OIDC client secret could fetch every agent credential
in the swarm; the one fetching this hive's matrix token could fetch
Grafana's. Least privilege was not misconfigured here, it was
unrepresentable.

Each now holds a leaf, a cert-auth role and a policy of its own, and each
policy is the single `secret/data/…` path that unit's own script names —
spelled to the leaf, not to a prefix, the way matrix-ctl's already is.
Following the four exemplars in-tree rather than building a mechanism:
`signLeaf` mints the leaves, `swarm-bao.nix` writes the roles from the
bootstrap token, the consumers name their own pair.

Two of the four are written PER HIVE and two are not, which is the shape
of the paths rather than a preference. A matrix appservice token and a
queue credential live under `swarm/hives/<name>/` and every hive runs a
reader for its own, so one role for all of them would have to be granted
`hives/*` — letting one hive read another's, a reach no hive has today.
An OIDC client secret lives under `swarm/services/<client-id>/` and a
swarm registers each exactly once, so one role each is enough. The
per-hive subjects are `<prefix>-<hive>` and swarm.nix reserves every
composed spelling as a hive name, so a hive cannot be named into another
hive's role.

The shared leaf stays: hive-c0re still passes it into its container, the
`bao` CLI wrapper still defaults to it, and the three
`glue-*-bao-identity.nix` files derive the PKI directory from it.

module-eval-bao-grants gains a negative arm per principal — each pins the
three stanzas the hive's leaf carried and the two wildcards a later
widening would reach for, so a policy that grows fails here rather than
in a store. Plus the consuming side: repointing a unit back at the hive's
leaf would evaluate, deploy and log in, and silently restore the union.

A hive that reads a store on another machine now places one leaf per
principal instead of one shared by four. That cost is the point, and
docs/swarm/secrets.md lists the pairs.
This commit is contained in:
atlas 2026-09-21 23:44:59 +02:00 committed by mara
commit f1445b4c8b
14 changed files with 859 additions and 43 deletions

View file

@ -45,6 +45,18 @@ let
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
deploy.bao.clientCaFile = lib.mkForce null;
};
# The store plus every one of the four readers that used to log in as the
# hive. One fixture rather than four: the claim they are four *separate*
# principals is only testable where all four render at once — that is the
# deployment in which two of them sharing a leaf would be invisible.
baoGrantWithConsumers = hive {
deploy.bao.enable = true;
deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token";
deploy.matrix.enable = true;
deploy.grafana.enable = true;
deploy.swarm-otel.enable = true;
};
cases = [
{
# Reads the rendered unit on the HOST, which is where the write happens:
@ -196,6 +208,250 @@ let
)
&& !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-matrix-ctl-policy);
}
# ── the four readers that used to share the hive's own leaf ──────────────
#
# 🩸 Until this split all four presented `deploy.bao.clientCertFile`, whose
# policy grants read on `swarm/agents/*`, `swarm/hives/<hive>/*` AND
# `swarm/services/*`. Four principals behind one certificate are one
# principal to bao, so the only expressible grant was the union: the unit
# fetching Grafana's OIDC secret could fetch every agent credential in the
# swarm.
#
# Every one of these cases carries the same three negative arms, and they
# are the deliverable rather than decoration — a positive arm alone passes
# just as well when the other two stanzas are still there beside it. The
# arms pin what each principal must NOT reach, so a later widening fails
# here instead of being noticed in a store.
{
name = "the matrix-token reader's grant is one hive's appservice token and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-matrix-token-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/h1/matrix/appservice-token\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
# The three stanzas the hive's own leaf carried, none of which this
# principal needs: every agent's credential, every service's OIDC
# client, and the rest of its own hive's tree — including the queue
# credential its sibling reader fetches.
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/services" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/*" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/queue" s)
# A `hives/*` wildcard would serve every hive from one role and let any
# hive read any other's token — the reach this split exists to remove,
# not to create.
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
# Nothing may rewrite the policy constraining it, for the reason the
# controller's own `hive-*` narrowing above gives.
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
name = "the queue-credential reader's grant is one hive's queue credential and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-queue-agent-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/hives/h1/queue/agent\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/services" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/*" s)
&& !(lib.hasInfix "secret/data/swarm/hives/h1/matrix" s)
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# ⚠️ The client id is the path segment, so the negative arm that matters
# for this one is the OTHER service's: `services/*` would have granted
# both, and the two are separate principals precisely because a
# dashboard is not entitled to a collector's credential.
name = "the Grafana OIDC reader's grant is Grafana's own client secret and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-grafana-oidc-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/swarm-grafana/oidc/client\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-collector" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# The mirror of the case above, and the arm naming `swarm-grafana` is why
# these are two principals rather than one `services/*` grant shared.
name = "the collector OIDC reader's grant is the collector's own client secret and nothing else";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-otel-oidc-policy.script;
in
lib.hasInfix "path \"secret/data/swarm/services/swarm-collector/oidc/client\" {" s
&& lib.hasInfix "capabilities = [\"read\"]" s
&& !(lib.hasInfix "secret/data/swarm/agents" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
&& !(lib.hasInfix "swarm-grafana" s)
&& !(lib.hasInfix "sys/policies/acl" s);
}
{
# 🩸 The half that makes the policies above bind: a policy grants only
# through a token that carries it, and a token is minted by a cert-auth
# role matching a CN. Four distinct subjects is the whole mechanism — one
# subject for four readers is one principal however the policies read.
#
# The per-hive subjects carry the hive name because their paths do; the
# two service subjects do not, because an OIDC client is registered once
# per swarm. Pinned so neither shape is tidied into the other.
name = "each of the four readers logs in under a subject of its own";
ok =
let
subjectOf =
unit: role: cn:
let
s = baoGrantHere.systemd.services.${unit}.script;
in
lib.hasInfix "auth/cert/certs/${role}" s
&& lib.hasInfix "allowed_common_names=${cn}" s
&& lib.hasInfix "token_policies=${role}" s
# Outside the `hive-*` namespace the controller may rewrite, for
# the reason the three service principals above state.
&& !(lib.hasInfix "auth/cert/certs/hive-" s);
in
subjectOf "swarm-bao-matrix-token-policy" "swarm-matrix-token-h1" "swarm-bao-matrix-token-h1"
&& subjectOf "swarm-bao-queue-agent-policy" "swarm-queue-agent-h1" "swarm-bao-queue-agent-h1"
&& subjectOf "swarm-bao-grafana-oidc-policy" "swarm-grafana-oidc" "swarm-bao-grafana-oidc"
&& subjectOf "swarm-bao-otel-oidc-policy" "swarm-otel-oidc" "swarm-bao-otel-oidc";
}
{
# 🩸 The consuming side, and the arm that would catch the regression that
# costs the most: a unit repointed back at `deploy.bao.clientCertFile`
# evaluates, deploys and logs in — and silently restores the union grant,
# because bao would again see one principal. Nothing about the policies
# above would look wrong.
#
# Each pair is asserted whole: a certificate with no key authenticates
# nothing, so a half-set pair is a reader that does not render.
name = "each of the four readers presents its own leaf, never the hive's";
ok =
let
b = baoGrantWithConsumers.services.hyperhive.deploy.bao;
hiveLeaf = [
b.clientCertFile
b.clientKeyFile
];
own = [
b.matrixTokenClientCertFile
b.matrixTokenClientKeyFile
b.queueAgentClientCertFile
b.queueAgentClientKeyFile
b.grafanaOidcClientCertFile
b.grafanaOidcClientKeyFile
b.otelOidcClientCertFile
b.otelOidcClientKeyFile
];
envOf = unit: baoGrantWithConsumers.systemd.services.${unit}.environment;
presents =
unit: cert: key:
(envOf unit).BAO_CLIENT_CERT == cert && (envOf unit).BAO_CLIENT_KEY == key;
in
lib.all (p: p != null) own
&& !(lib.any (p: lib.elem p hiveLeaf) own)
&& lib.length (lib.unique own) == lib.length own
&& presents "swarm-bao-matrix-token" b.matrixTokenClientCertFile b.matrixTokenClientKeyFile
&& presents "swarm-bao-queue-agent" b.queueAgentClientCertFile b.queueAgentClientKeyFile
&& presents "swarm-bao-grafana-oidc" b.grafanaOidcClientCertFile b.grafanaOidcClientKeyFile
&& presents "swarm-bao-otel-oidc" b.otelOidcClientCertFile b.otelOidcClientKeyFile;
}
{
# The minting side of the same claim. A role matching a subject nothing
# signs is a reader that cannot log in, so the leaves and the roles have
# to be asserted against each other — and the two per-hive leaves carry
# THIS host's hive name, which is what makes one hive's leaf useless
# against another hive's role.
name = "the PKI unit signs a leaf per reader, each under that reader's own subject";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-pki.script;
in
# The basename and the subject are matched separately: `signLeaf` takes
# them as consecutive arguments across a `\` continuation, so one
# literal spanning both would pin this file's line wrapping rather than
# the pairing it means to.
lib.all (lib.flip lib.hasInfix s) [
"/matrix-token.pem ]"
"swarm-bao-matrix-token-h1 \"\" clientAuth"
"/queue-agent.pem ]"
"swarm-bao-queue-agent-h1 \"\" clientAuth"
"/grafana-oidc.pem ]"
"swarm-bao-grafana-oidc \"\" clientAuth"
"/otel-oidc.pem ]"
"swarm-bao-otel-oidc \"\" clientAuth"
];
}
{
# The absence arm: with no client CA there is no trust anchor, so the
# login roles cannot be written — but the policies they would attach are
# still asserted, exactly as the three service principals above behave in
# this deployment. A unit that vanished here would take the policy with
# it and leave nothing to diagnose.
name = "with no client CA the four readers get policies but no login roles";
ok =
let
units = [
"swarm-bao-matrix-token-policy"
"swarm-bao-queue-agent-policy"
"swarm-bao-grafana-oidc-policy"
"swarm-bao-otel-oidc-policy"
];
scriptOf = unit: baoGrantNoClientCa.systemd.services.${unit}.script;
in
lib.all (
unit:
(baoGrantNoClientCa.systemd.services ? ${unit})
&& lib.hasInfix "bao policy write" (scriptOf unit)
&& !(lib.hasInfix "auth/cert/certs" (scriptOf unit))
) units;
}
{
# Same control the three service principals carry: the write needs a
# client certificate and the host is the side that has one, so a unit
# rendered inside the store's container would have neither an identity
# nor a route. Plus the ordering that makes the mounts exist first.
name = "the four readers' granting units are ordered after the mounts and rendered on the host";
ok =
let
units = [
"swarm-bao-matrix-token-policy"
"swarm-bao-queue-agent-policy"
"swarm-bao-grafana-oidc-policy"
"swarm-bao-otel-oidc-policy"
];
in
lib.all (
unit:
lib.elem "swarm-bao-controller-policy.service" baoGrantHere.systemd.services.${unit}.after
&& !(baoGrantHere.containers.swarm-bao.config.systemd.services ? ${unit})
) units;
}
{
# A store host that has not placed a bootstrap token can write no grant at
# all, so none of the four units may exist — the same claim
# `baoGrantNoStore` makes for the controller's, one file over. Without
# this arm `lib.mkIf haveBootstrapToken` could be dropped from the shared
# builder and every other case here would still pass.
name = "without a bootstrap token none of the four readers' granting units render";
ok =
let
s = baoGrantNoStore.systemd.services;
in
!(s ? swarm-bao-matrix-token-policy)
&& !(s ? swarm-bao-queue-agent-policy)
&& !(s ? swarm-bao-grafana-oidc-policy)
&& !(s ? swarm-bao-otel-oidc-policy);
}
{
# The policy authorising this route lives in another file, and nothing
# else relates the grants to the paths the code actually writes.

View file

@ -32,10 +32,20 @@ let
# nothing here mints a leaf and the operator names one placed by hand. The
# deployment this pairing exists to serve, and the one that was previously
# inexpressible — the gate asked whether the store was a neighbour.
#
# 🩸 One leaf PER READER, and this fixture is where that cost is visible: the
# two units below identify themselves to the store separately, so an operator
# placing leaves by hand places one for each rather than one for both. Naming
# only `clientCertFile` here would leave neither unit rendered — which is what
# the arms below would then be asserting about.
baoRemoteReader = hive {
deploy.matrix.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.matrixTokenClientCertFile = "/etc/pki/bao-matrix-token.pem";
deploy.bao.matrixTokenClientKeyFile = "/etc/pki/bao-matrix-token-key.pem";
deploy.bao.queueAgentClientCertFile = "/etc/pki/bao-queue-agent.pem";
deploy.bao.queueAgentClientKeyFile = "/etc/pki/bao-queue-agent-key.pem";
};
# A homeserver on a hive with NO store identity at all — neither a local

View file

@ -41,6 +41,8 @@ let
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
};
# The same UI with the IdP on ANOTHER host and a store leaf placed by hand.
@ -54,6 +56,8 @@ let
swarm.authelia.url = "https://auth.example.invalid";
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.grafanaOidcClientCertFile = "/etc/pki/bao-grafana-oidc.pem";
deploy.bao.grafanaOidcClientKeyFile = "/etc/pki/bao-grafana-oidc-key.pem";
};
# A Grafana host holding no store identity. This used to be the shape the
@ -148,8 +152,8 @@ let
# mean the two assertions had been collapsed into one conjunction.
name = "a grafana host with no store identity is refused, naming the options to set";
ok =
grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile"
&& grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile"
grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientCertFile"
&& grafanaRefusedFor grafanaNoIdentity "deploy.bao.grafanaOidcClientKeyFile"
&& !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url");
}
{
@ -161,9 +165,9 @@ let
name = "neither grafana refusal fires on a correctly configured host, co-located or not";
ok =
!(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url")
&& !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile")
&& !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.grafanaOidcClientCertFile")
&& !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url")
&& !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile");
&& !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.grafanaOidcClientCertFile");
}
{
# Same 403-not-a-miss reason as the matrix and queue arms below: the

View file

@ -63,6 +63,35 @@ let
swarm.hives.mintctl.domain = "m.t.local";
};
# The two OIDC-secret readers' subjects, fixed strings like the three above.
hiveNamedAfterGrafanaOidcSubject = hive {
deploy.swarm-otel.enable = false;
deploy.bao.grafanaOidcCommonName = "gfctl";
swarm.hives.gfctl.domain = "g.t.local";
};
hiveNamedAfterOtelOidcSubject = hive {
deploy.swarm-otel.enable = false;
deploy.bao.otelOidcCommonName = "otctl";
swarm.hives.otctl.domain = "o.t.local";
};
# 🩸 A different shape from every fixture above: the matrix-token and
# queue-credential roles are written PER HIVE, so the subject a hive must not
# be is `<prefix>-<some hive's name>` rather than the prefix itself. Reserving
# only the prefix would leave the composed spelling free, and a hive taking it
# would present a leaf the other hive's role accepts — which is a hive reading
# another hive's queue credential, the exact widening the split exists to
# avoid.
#
# Two hives here, not one: the collision is with the OTHER hive's role.
hiveNamedAfterPerHiveReaderSubject = hive {
deploy.swarm-otel.enable = false;
deploy.bao.queueAgentCommonNamePrefix = "qr";
swarm.hives.other.domain = "o.t.local";
swarm.hives.qr-other.domain = "q.t.local";
};
hiveNameWithComposedWord = hive {
deploy.swarm-otel.enable = false;
swarm.hives."h1-agent".domain = "a.t.local";
@ -111,6 +140,36 @@ let
a: !a.assertion && lib.hasInfix "'mintctl'" a.message
) hiveNamedAfterMatrixCtlSubject.assertions;
}
{
# The fourth and fifth, for the reason the case above gives: `certAuthCns`
# is where a role added beside the others registers itself, and nothing
# but a case per element notices when one forgets. These two are the
# subjects of the readers that fetch Grafana's and the collector's OIDC
# client secrets.
name = "a hive named after either OIDC-secret reader's subject is refused too";
ok =
equalityGuardFired hiveNamedAfterGrafanaOidcSubject
&& lib.any (
a: !a.assertion && lib.hasInfix "'gfctl'" a.message
) hiveNamedAfterGrafanaOidcSubject.assertions
&& equalityGuardFired hiveNamedAfterOtelOidcSubject
&& lib.any (
a: !a.assertion && lib.hasInfix "'otctl'" a.message
) hiveNamedAfterOtelOidcSubject.assertions;
}
{
# 🩸 The per-hive half, and the one a prefix-only reservation would miss:
# the role is `<prefix>-<hive>`, so the reserved string has to be composed
# against every declared hive. Here hive `qr-other` collides with the role
# written for hive `other` — a leaf that reads a credential belonging to a
# hive that is not it.
name = "a hive named after another hive's per-hive reader subject is refused";
ok =
equalityGuardFired hiveNamedAfterPerHiveReaderSubject
&& lib.any (
a: !a.assertion && lib.hasInfix "'qr-other'" a.message
) hiveNamedAfterPerHiveReaderSubject.assertions;
}
{
# Without this the case above proves nothing: an arm that fires for every
# roster is not a guard, and `hives` is non-empty in both fixtures.

View file

@ -32,6 +32,8 @@ let
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
deploy.victoriametrics.enable = false;
deploy.victorialogs.enable = false;
};
@ -44,6 +46,8 @@ let
deploy.authelia.enable = true;
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
};
# The same collector with the IdP on ANOTHER host and a store leaf placed by
@ -54,6 +58,8 @@ let
swarm.authelia.url = "https://auth.example.invalid";
deploy.bao.clientCertFile = "/etc/pki/bao-client.pem";
deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem";
deploy.bao.otelOidcClientCertFile = "/etc/pki/bao-otel-oidc.pem";
deploy.bao.otelOidcClientKeyFile = "/etc/pki/bao-otel-oidc-key.pem";
};
cases = [
{