vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-bao: the responder may create the callout xkey; every token reads swarm/public/

Browse source 
The swarm-nats-auth role gains create+read on exactly the callout xkey's
two leaves, and no update, so a stored pair is never replaced.

swarm/public/ is readable by every token through bao's default policy.
swarm-bao-public-read-policy reads that policy, replaces only its own
marker-delimited block, and writes it back when it differs; it never
writes a policy it read back empty. Writing `default` is a new grant on
the granter's own policy, so on an existing store it lands only after
the bootstrap-token step re-runs; until then the unit says so and exits
0, and swarm-bao-granter-role restarts it on success.
This commit is contained in:
atlas 2026-10-05 21:25:40 +02:00 • committed by mara
commit bb571285fd
3 changed files with 258 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

135
nix/host-modules/swarm-bao.nix
View file

@ -185,9 +185,11 @@ let
# the services root. The next six set up the agent PKI mount: the mount, its
# root and the `swarm-*` role agent certificates are issued through. No
# `root` delete there: every agent's cert-auth role pins that root by value,
# so replacing it would lock every agent out. The last three configure the
# so replacing it would lock every agent out. The next three configure the
# `oidc` auth method: its config, its `swarm-*` roles, and a read on the one
# client secret that config carries. No `sys/auth`: the auth mounts are
# client secret that config carries. The last is bao's `default` policy,
# which every token carries: `swarm-bao-public-read-policy` adds
# the `swarm/public/` read to it. No `sys/auth`: the auth mounts are
# created with the bootstrap token by `swarm-bao-granter-role`.
#
# Piped as a shell-quoted argument like `controllerPolicyText`, so
@ -272,6 +274,10 @@ let
path "${credentialMountPath}/data/${baoUiClientLeaf}" {
capabilities = ["read"]
}
path "sys/policies/acl/default" {
capabilities = ["create", "read", "update"]
}
'';
# What a granting unit prints when the store refuses the granter: the
@ -849,16 +855,49 @@ let
# one path segment, where `*` globs only at the end and would reach every
# other credential an agent holds. Plus the responder's own OIDC client
# secret, which `swarm-bao-nats-auth-oidc` reads under this same leaf.
#
# And the callout xkey's two leaves, which `ensure-xkey` creates when the
# store holds no seed. `create` without `update`, so a stored pair is never
# replaced; `read` because creating is read-then-create, and the responder
# loads the seed as it starts.
natsAuthReaders = [
{
name = "swarm-nats-auth";
cn = baoDeploy.natsAuthCommonName;
policyText =
readStanza "${credentialMountPath}/data/swarm/agents/+/queue"
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.nats.clientId}/oidc/client";
+ readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.nats.clientId}/oidc/client"
+ createStanza "${credentialMountPath}/data/${xkeySeedLeaf}"
+ createStanza "${credentialMountPath}/data/${xkeyPublicLeaf}";
}
];
# The callout xkey, the nix half of `swarm-nats-auth`'s `xkey::SEED_PATH`
# and `xkey::PUBLIC_PATH`. The seed is the responder's alone; the public half
# is under `swarm/public/`, which `publicReadBlock` lets every token read.
xkeySeedLeaf = "swarm/services/swarm-nats/callout/xkey-seed";
xkeyPublicLeaf = "swarm/public/swarm-nats/callout-xkey";
createStanza = path: ''
path "${path}" {
capabilities = ["create", "read"]
}
'';
# What `swarm/public/` is: readable by every token the store issues, through
# bao's `default` policy, which every token carries. A block between two
# marker lines, so `swarm-bao-public-read-policy` can replace it and leave
# the rest of that policy, the store's own, as it found it.
publicReadBegin = "# hyperhive: swarm/public begin";
publicReadEnd = "# hyperhive: swarm/public end";
publicReadBlock = lib.concatStringsSep "\n" [
publicReadBegin
''path "${credentialMountPath}/data/swarm/public/*" {''
" capabilities = [\"read\"]"
"}"
publicReadEnd
];
# The CI runner's token fetch. The one reader of the path, which the
# controller writes and cannot read back.
ciRunnerReaders = [
@ -2535,12 +2574,12 @@ in
Restart = "on-failure";
RestartSec = 30;
# Runs only once the script above exits 0, which is when the
# granter may first configure `auth/oidc`; the viewer unit exits 0
# without doing so before then, and nothing else re-runs it. A
# restart because it is `RemainAfterExit`, where a start is a
# no-op.
# granter may first configure `auth/oidc` and write bao's `default`
# policy; the viewer and public-read units exit 0 without doing so
# before then, and nothing else re-runs them. A restart because
# they are `RemainAfterExit`, where a start is a no-op.
#
# `--no-block`: the viewer unit is ordered after this one, so a
# `--no-block`: both units are ordered after this one, so a
# blocking restart waits on a job that cannot run until this unit
# is active, which it is not until this command returns.
#
@ -2549,7 +2588,7 @@ in
# already running (or a `systemctl` that itself fails to enqueue
# the job) would mark the granter failed and trigger its own
# `Restart=on-failure` for a problem that isn't the granter's.
ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service";
ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service swarm-bao-public-read-policy.service";
};
script = ''
set -euo pipefail
@ -3380,6 +3419,84 @@ in
'';
};
# `publicReadBlock` in bao's `default` policy. Read, the old block
# dropped, the current one appended, and written back only when that
# differs, so the store's own stanzas stay as the store wrote them.
#
# The granter may write `default` only once `swarm-bao-granter-role` has
# run with a bootstrap token since that grant was added. Until then this
# says so and exits 0, as the viewer unit does: principals granted a
# `swarm/public/` path by name still read it, and a day of retries would
# change nothing. `swarm-bao-granter-role` restarts this unit each time
# it succeeds.
systemd.services.swarm-bao-public-read-policy = lib.mkIf haveGranter {
description = "let every bao token read swarm/public/ through the default policy";
after = [
"container@${cfg.machine}.service"
"swarm-bao-controller-policy.service"
]
++ granterAfter;
requires = [ "swarm-bao-pki.service" ];
wantedBy = [ "multi-user.target" ];
path = [
baoCli
pkgs.coreutils
pkgs.gawk
];
environment = granterEnv;
# Same unseal wait as its siblings above, for the reason stated there.
startLimitBurst = 2880;
startLimitIntervalSec = 90000;
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
Restart = "on-failure";
RestartSec = 30;
};
script = ''
set -euo pipefail
${granterLogin}
caps="$(bao token capabilities sys/policies/acl/default)"
case "$caps" in
*update*) ;;
*)
echo "the granter may not write the store's default policy yet (capabilities: $caps), so swarm/public/ is readable only where a policy names it." >&2
echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2
${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps}
echo "swarm-bao-granter-role re-runs this unit once it succeeds." >&2
exit 0
;;
esac
if ! current="$(bao read -field=policy sys/policies/acl/default 2>"$err")"; then
echo "could not read the store's default policy; leaving it as it is." >&2
cat "$err" >&2
exit 1
fi
# Never write a default policy built from nothing: it would drop
# every stanza the store's tokens rely on for their own lookups.
if [ -z "$current" ]; then
echo "the store returned an empty default policy; leaving it as it is." >&2
exit 1
fi
kept="$(printf '%s\n' "$current" |
awk -v b=${lib.escapeShellArg publicReadBegin} -v e=${lib.escapeShellArg publicReadEnd} \
'$0 == b { skip = 1 } !skip { print } $0 == e { skip = 0 }')"
wanted="$kept
${publicReadBlock}"
if [ "$wanted" = "$current" ]; then
echo "the default policy already lets every token read swarm/public/"
exit 0
fi
printf '%s\n' "$wanted" |
bao policy write default -
echo "the default policy now lets every token read swarm/public/"
'';
};
# The CA bind source is written at runtime by a host unit, so the
# container has to start after it — otherwise nspawn sets up a mount
# over a file that does not exist yet.

130
nix/module-eval/bao-grants.nix
View file

@ -170,8 +170,8 @@ let
_: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile
) baoGrantWithConsumers.systemd.services;
# The fourteen units that write a `swarm-*` grant, by name, for the discovery
# control below.
# The fifteen units that write a grant as the granter, by name, for the
# discovery control below.
grantingUnitNames = [
"swarm-bao-controller-policy"
"swarm-bao-secret-publisher-policy"
@ -187,6 +187,7 @@ let
"swarm-bao-operator-viewer-policy"
"swarm-bao-hive-leaf-policy"
"swarm-bao-ci-runner-policy"
"swarm-bao-public-read-policy"
];
# Comment lines dropped first: both the HCL and the scripts explain
@ -573,16 +574,33 @@ let
# `+` is one path segment, so this reaches `swarm/agents/<agent>/queue`
# and no other credential an agent holds; `swarm/agents/*` would reach
# all of them. The responder's own OIDC client secret is the one
# `services/` leaf beside it.
name = "the queue responder's grant is every agent's queue credential and its own OIDC secret, nothing else";
# `services/` leaf beside it. Its only creates are the callout xkey's
# two leaves, and nothing it holds may update or delete one.
name = "the queue responder reads agent queue credentials and its OIDC secret, and may create only the callout xkey's two leaves";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-nats-auth-policy.script;
grants = grantsIn s;
creates = map (g: g.path) (lib.filter (g: lib.elem "create" g.caps) grants);
in
lib.hasInfix "path \"secret/data/swarm/agents/+/queue\" {" s
&& lib.hasInfix "path \"secret/data/swarm/services/swarm-nats/oidc/client\" {" s
&& lib.length (lib.filter lib.isList (builtins.split "capabilities = \\[\"read\"]" s)) == 2
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2
&&
lib.sort lib.lessThan creates == [
"secret/data/swarm/public/swarm-nats/callout-xkey"
"secret/data/swarm/services/swarm-nats/callout/xkey-seed"
]
&& lib.all (
g:
g.caps == [ "read" ]
||
g.caps == [
"create"
"read"
]
) grants
&& lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 4
&& !(lib.hasInfix "secret/data/swarm/agents/*" s)
&& !(lib.hasInfix "secret/data/swarm/hives" s)
&& !(lib.hasInfix "secret/data/swarm/services/*" s)
@ -795,24 +813,24 @@ let
}
{
# A store host without the granter's pair writes its grants some other
# way, so none of the fourteen units may exist. Without this arm
# way, so none of the fifteen units may exist. Without this arm
# `lib.mkIf haveGranter` could be dropped from any of them and every other
# case here would still pass.
name = "without the granter's pair none of the fourteen granting units render";
name = "without the granter's pair none of the fifteen granting units render";
ok =
let
s = baoGranterOptOut.systemd.services;
in
lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ])
# The control: the same store with the pair renders all fourteen.
# The control: the same store with the pair renders all fifteen.
&& lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames;
}
{
# 🩸 What replaced the silent skip. With no bootstrap token the fourteen still
# 🩸 What replaced the silent skip. With no bootstrap token the fifteen still
# render, and a refused granter fails them with the step that fixes it.
# A store host that never named a token is told to name one, since the
# unit that sets the granter up renders only where it has.
name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step";
name = "a store host without a bootstrap token renders the fifteen, each failing loudly with the one-time step";
ok =
let
s = baoGranterNoToken.systemd.services;
@ -919,7 +937,7 @@ let
{
# The granter's grants, whole. Pinned as the full list, because an added
# path or capability is exactly what a presence check misses.
name = "the granter's policy is exactly these twenty stanzas";
name = "the granter's policy is exactly these twenty-one stanzas";
ok =
let
cu = [
@ -1011,6 +1029,14 @@ let
path = "secret/data/swarm/services/swarm-bao-ui/oidc/client";
caps = [ "read" ];
}
{
path = "sys/policies/acl/default";
caps = [
"create"
"read"
"update"
];
}
];
}
{
@ -1025,9 +1051,10 @@ let
];
}
{
# Outside `swarm-*` and the store's own mounts it holds nothing: no
# hive's policy or role, no auth mount, no token, no secret.
name = "the granter grants nothing outside swarm-* and the store's own mounts";
# Outside `swarm-*`, the store's own mounts and the `default` policy it
# holds nothing: no hive's policy or role, no auth mount, no token, no
# secret.
name = "the granter grants nothing outside swarm-*, the store's own mounts and the default policy";
ok =
lib.all (p: grantFor granterGrants p == null) [
"sys/policies/acl/hive-x"
@ -1443,8 +1470,8 @@ let
}
{
# What makes the case above mean something: discovery by the granter's
# certificate reaches all fourteen units, and each yields calls.
name = "the granter-policy check sees all fourteen granting units, and parses calls from each";
# certificate reaches all fifteen units, and each yields calls.
name = "the granter-policy check sees all fifteen granting units, and parses calls from each";
ok =
lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames
&& lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits)
@ -1706,6 +1733,77 @@ let
&& !(lib.elem "swarm-bao-operator-viewer-policy.service" g.after)
&& !(v ? postStart && lib.hasInfix "swarm-bao-granter-role" v.postStart);
}
{
# Every token carries bao's `default` policy, so a read added there is
# a read for every authenticated client. Exactly one stanza, a read on
# `swarm/public/` and nothing wider; the markers are what a later run
# replaces it by.
name = "the default policy gains one read, on swarm/public/, between markers";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-public-read-policy.script;
block = lib.head (
lib.tail (
lib.splitString "# hyperhive: swarm/public begin\n" (builtins.unsafeDiscardStringContext s)
)
);
stanzas = grantsIn (lib.head (lib.splitString "# hyperhive: swarm/public end" block));
in
stanzas == [
{
path = "secret/data/swarm/public/*";
caps = [ "read" ];
}
]
&& lib.hasInfix "-v e='# hyperhive: swarm/public end'" s
&& lib.hasInfix "printf '%s\\n' \"$wanted\" |\n bao policy write default -" s;
}
{
# A read-modify-write of a policy every token depends on: the read
# comes first, an empty one stops the unit before anything is written,
# and the capability probe comes before both, so a granter without the
# grant writes nothing.
name = "the default policy is read before it is written, and never written from an empty read";
ok =
let
s = builtins.unsafeDiscardStringContext baoGrantHere.systemd.services.swarm-bao-public-read-policy.script;
at = needle: lib.stringLength (lib.head (lib.splitString needle s));
probe = "caps=\"$(bao token capabilities sys/policies/acl/default)\"";
read = "bao read -field=policy sys/policies/acl/default";
emptyGuard = "if [ -z \"$current\" ]; then";
write = "bao policy write default -";
in
lib.hasInfix probe s
&& at probe < at "exit 0"
&& at "exit 0" < at read
&& at read < at emptyGuard
&& at emptyGuard < at write
&& lib.hasInfix "swarm-bao-granter-role re-runs this unit once it succeeds." s;
}
{
# The granter's own policy is what lets that unit write `default`, and a
# changed granter policy lands only with the bootstrap step, so the
# step's success restarts the unit as it does the viewer's.
name = "the granter may write the default policy, and its step restarts the public-read unit";
ok =
let
s = baoGrantHere.systemd.services;
g = s.swarm-bao-granter-role;
p = s.swarm-bao-public-read-policy;
in
lib.any (
gr:
gr.path == "sys/policies/acl/default"
&&
gr.caps == [
"create"
"read"
"update"
]
) granterGrants
&& lib.hasInfix "swarm-bao-public-read-policy.service" g.serviceConfig.ExecStartPost
&& lib.elem "swarm-bao-granter-role.service" p.after;
}
{
# The client secret is on stdin, never an argument in /proc; the rest is
# the swarm's authelia and the UI's own client id.