From bb571285fd10a85f652a5f2016ff10d734a426bb Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 5 Oct 2026 21:25:40 +0200 Subject: [PATCH] swarm-bao: the responder may create the callout xkey; every token reads swarm/public/ The swarm-nats-auth role gains create+read on exactly the callout xkey's two leaves, and no update, so a stored pair is never replaced. swarm/public/ is readable by every token through bao's default policy. swarm-bao-public-read-policy reads that policy, replaces only its own marker-delimited block, and writes it back when it differs; it never writes a policy it read back empty. Writing `default` is a new grant on the granter's own policy, so on an existing store it lands only after the bootstrap-token step re-runs; until then the unit says so and exits 0, and swarm-bao-granter-role restarts it on success. --- docs/swarm/bao.md | 21 ++++- nix/host-modules/swarm-bao.nix | 135 ++++++++++++++++++++++++++++++--- nix/module-eval/bao-grants.nix | 130 +++++++++++++++++++++++++++---- 3 files changed, 258 insertions(+), 28 deletions(-) diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index e73c97f2..f5abcc61 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -55,9 +55,9 @@ the unit's journal instead. ### Residual risk -The granter is root-equivalent. It may write any `swarm-*` policy with any -content, and attach it to a role that accepts any certificate; no bao ACL can -constrain what a policy says. What bounds it: +The granter is root-equivalent. It may write any `swarm-*` policy, and bao's +`default` policy, with any content, and attach a policy to a role that accepts +any certificate; no bao ACL can constrain what a policy says. What bounds it: - `nix/host-modules/swarm-bao.nix` renders every policy it writes, and module-eval pins each principal's grants. **Merging a change to that policy @@ -76,6 +76,21 @@ constrain what a policy says. What bounds it: restarting `swarm-bao-pki` mints a new leaf, but doesn't invalidate the old one. +### What every token may read + +Anything under `secret/swarm/public/` is readable by every token the store +issues. `swarm-bao-public-read-policy` puts that one read into bao's `default` +policy, which every token carries, between two marker lines. It reads the +policy, replaces only its own block, and writes the result back when it +differs, so the store's own stanzas stay as the store wrote them. It never +writes a policy it read back empty. The callout xkey's public half lives +there ([`secrets.md`](secrets.md#the-queues-callout-xkey)). + +Writing `default` is a grant on the granter's own policy, so on a store +whose granter predates it, the unit says so and exits 0 until the bootstrap +step re-runs. Until then only principals whose policy names a `swarm/public/` +path read it; the queue's responder is one, so the queue doesn't wait on this. + ## Sealing `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index aa6d3d42..5775f3eb 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -185,9 +185,11 @@ let # the services root. The next six set up the agent PKI mount: the mount, its # root and the `swarm-*` role agent certificates are issued through. No # `root` delete there: every agent's cert-auth role pins that root by value, - # so replacing it would lock every agent out. The last three configure the + # so replacing it would lock every agent out. The next three configure the # `oidc` auth method: its config, its `swarm-*` roles, and a read on the one - # client secret that config carries. No `sys/auth`: the auth mounts are + # client secret that config carries. The last is bao's `default` policy, + # which every token carries: `swarm-bao-public-read-policy` adds + # the `swarm/public/` read to it. No `sys/auth`: the auth mounts are # created with the bootstrap token by `swarm-bao-granter-role`. # # Piped as a shell-quoted argument like `controllerPolicyText`, so @@ -272,6 +274,10 @@ let path "${credentialMountPath}/data/${baoUiClientLeaf}" { capabilities = ["read"] } + + path "sys/policies/acl/default" { + capabilities = ["create", "read", "update"] + } ''; # What a granting unit prints when the store refuses the granter: the @@ -849,16 +855,49 @@ let # one path segment, where `*` globs only at the end and would reach every # other credential an agent holds. Plus the responder's own OIDC client # secret, which `swarm-bao-nats-auth-oidc` reads under this same leaf. + # + # And the callout xkey's two leaves, which `ensure-xkey` creates when the + # store holds no seed. `create` without `update`, so a stored pair is never + # replaced; `read` because creating is read-then-create, and the responder + # loads the seed as it starts. natsAuthReaders = [ { name = "swarm-nats-auth"; cn = baoDeploy.natsAuthCommonName; policyText = readStanza "${credentialMountPath}/data/swarm/agents/+/queue" - + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.nats.clientId}/oidc/client"; + + readStanza "${credentialMountPath}/data/swarm/services/${hyperhiveCfg.swarm.nats.clientId}/oidc/client" + + createStanza "${credentialMountPath}/data/${xkeySeedLeaf}" + + createStanza "${credentialMountPath}/data/${xkeyPublicLeaf}"; } ]; + # The callout xkey, the nix half of `swarm-nats-auth`'s `xkey::SEED_PATH` + # and `xkey::PUBLIC_PATH`. The seed is the responder's alone; the public half + # is under `swarm/public/`, which `publicReadBlock` lets every token read. + xkeySeedLeaf = "swarm/services/swarm-nats/callout/xkey-seed"; + xkeyPublicLeaf = "swarm/public/swarm-nats/callout-xkey"; + + createStanza = path: '' + path "${path}" { + capabilities = ["create", "read"] + } + ''; + + # What `swarm/public/` is: readable by every token the store issues, through + # bao's `default` policy, which every token carries. A block between two + # marker lines, so `swarm-bao-public-read-policy` can replace it and leave + # the rest of that policy, the store's own, as it found it. + publicReadBegin = "# hyperhive: swarm/public begin"; + publicReadEnd = "# hyperhive: swarm/public end"; + publicReadBlock = lib.concatStringsSep "\n" [ + publicReadBegin + ''path "${credentialMountPath}/data/swarm/public/*" {'' + " capabilities = [\"read\"]" + "}" + publicReadEnd + ]; + # The CI runner's token fetch. The one reader of the path, which the # controller writes and cannot read back. ciRunnerReaders = [ @@ -2535,12 +2574,12 @@ in Restart = "on-failure"; RestartSec = 30; # Runs only once the script above exits 0, which is when the - # granter may first configure `auth/oidc`; the viewer unit exits 0 - # without doing so before then, and nothing else re-runs it. A - # restart because it is `RemainAfterExit`, where a start is a - # no-op. + # granter may first configure `auth/oidc` and write bao's `default` + # policy; the viewer and public-read units exit 0 without doing so + # before then, and nothing else re-runs them. A restart because + # they are `RemainAfterExit`, where a start is a no-op. # - # `--no-block`: the viewer unit is ordered after this one, so a + # `--no-block`: both units are ordered after this one, so a # blocking restart waits on a job that cannot run until this unit # is active, which it is not until this command returns. # @@ -2549,7 +2588,7 @@ in # already running (or a `systemctl` that itself fails to enqueue # the job) would mark the granter failed and trigger its own # `Restart=on-failure` for a problem that isn't the granter's. - ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service"; + ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service swarm-bao-public-read-policy.service"; }; script = '' set -euo pipefail @@ -3380,6 +3419,84 @@ in ''; }; + # `publicReadBlock` in bao's `default` policy. Read, the old block + # dropped, the current one appended, and written back only when that + # differs, so the store's own stanzas stay as the store wrote them. + # + # The granter may write `default` only once `swarm-bao-granter-role` has + # run with a bootstrap token since that grant was added. Until then this + # says so and exits 0, as the viewer unit does: principals granted a + # `swarm/public/` path by name still read it, and a day of retries would + # change nothing. `swarm-bao-granter-role` restarts this unit each time + # it succeeds. + systemd.services.swarm-bao-public-read-policy = lib.mkIf haveGranter { + description = "let every bao token read swarm/public/ through the default policy"; + after = [ + "container@${cfg.machine}.service" + "swarm-bao-controller-policy.service" + ] + ++ granterAfter; + requires = [ "swarm-bao-pki.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ + baoCli + pkgs.coreutils + pkgs.gawk + ]; + environment = granterEnv; + # Same unseal wait as its siblings above, for the reason stated there. + startLimitBurst = 2880; + startLimitIntervalSec = 90000; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + Restart = "on-failure"; + RestartSec = 30; + }; + script = '' + set -euo pipefail + + ${granterLogin} + + caps="$(bao token capabilities sys/policies/acl/default)" + case "$caps" in + *update*) ;; + *) + echo "the granter may not write the store's default policy yet (capabilities: $caps), so swarm/public/ is readable only where a policy names it." >&2 + echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 + ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} + echo "swarm-bao-granter-role re-runs this unit once it succeeds." >&2 + exit 0 + ;; + esac + + if ! current="$(bao read -field=policy sys/policies/acl/default 2>"$err")"; then + echo "could not read the store's default policy; leaving it as it is." >&2 + cat "$err" >&2 + exit 1 + fi + # Never write a default policy built from nothing: it would drop + # every stanza the store's tokens rely on for their own lookups. + if [ -z "$current" ]; then + echo "the store returned an empty default policy; leaving it as it is." >&2 + exit 1 + fi + + kept="$(printf '%s\n' "$current" | + awk -v b=${lib.escapeShellArg publicReadBegin} -v e=${lib.escapeShellArg publicReadEnd} \ + '$0 == b { skip = 1 } !skip { print } $0 == e { skip = 0 }')" + wanted="$kept + ${publicReadBlock}" + if [ "$wanted" = "$current" ]; then + echo "the default policy already lets every token read swarm/public/" + exit 0 + fi + printf '%s\n' "$wanted" | + bao policy write default - + echo "the default policy now lets every token read swarm/public/" + ''; + }; + # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index e4079a24..5608ba8a 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -170,8 +170,8 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The fourteen units that write a `swarm-*` grant, by name, for the discovery - # control below. + # The fifteen units that write a grant as the granter, by name, for the + # discovery control below. grantingUnitNames = [ "swarm-bao-controller-policy" "swarm-bao-secret-publisher-policy" @@ -187,6 +187,7 @@ let "swarm-bao-operator-viewer-policy" "swarm-bao-hive-leaf-policy" "swarm-bao-ci-runner-policy" + "swarm-bao-public-read-policy" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -573,16 +574,33 @@ let # `+` is one path segment, so this reaches `swarm/agents//queue` # and no other credential an agent holds; `swarm/agents/*` would reach # all of them. The responder's own OIDC client secret is the one - # `services/` leaf beside it. - name = "the queue responder's grant is every agent's queue credential and its own OIDC secret, nothing else"; + # `services/` leaf beside it. Its only creates are the callout xkey's + # two leaves, and nothing it holds may update or delete one. + name = "the queue responder reads agent queue credentials and its OIDC secret, and may create only the callout xkey's two leaves"; ok = let s = baoGrantHere.systemd.services.swarm-bao-nats-auth-policy.script; + grants = grantsIn s; + creates = map (g: g.path) (lib.filter (g: lib.elem "create" g.caps) grants); in lib.hasInfix "path \"secret/data/swarm/agents/+/queue\" {" s && lib.hasInfix "path \"secret/data/swarm/services/swarm-nats/oidc/client\" {" s && lib.length (lib.filter lib.isList (builtins.split "capabilities = \\[\"read\"]" s)) == 2 - && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 2 + && + lib.sort lib.lessThan creates == [ + "secret/data/swarm/public/swarm-nats/callout-xkey" + "secret/data/swarm/services/swarm-nats/callout/xkey-seed" + ] + && lib.all ( + g: + g.caps == [ "read" ] + || + g.caps == [ + "create" + "read" + ] + ) grants + && lib.length (lib.filter lib.isList (builtins.split "path \"" s)) == 4 && !(lib.hasInfix "secret/data/swarm/agents/*" s) && !(lib.hasInfix "secret/data/swarm/hives" s) && !(lib.hasInfix "secret/data/swarm/services/*" s) @@ -795,24 +813,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the fourteen units may exist. Without this arm + # way, so none of the fifteen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the fourteen granting units render"; + name = "without the granter's pair none of the fifteen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all fourteen. + # The control: the same store with the pair renders all fifteen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the fourteen still + # 🩸 What replaced the silent skip. With no bootstrap token the fifteen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the fifteen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -919,7 +937,7 @@ let { # The granter's grants, whole. Pinned as the full list, because an added # path or capability is exactly what a presence check misses. - name = "the granter's policy is exactly these twenty stanzas"; + name = "the granter's policy is exactly these twenty-one stanzas"; ok = let cu = [ @@ -1011,6 +1029,14 @@ let path = "secret/data/swarm/services/swarm-bao-ui/oidc/client"; caps = [ "read" ]; } + { + path = "sys/policies/acl/default"; + caps = [ + "create" + "read" + "update" + ]; + } ]; } { @@ -1025,9 +1051,10 @@ let ]; } { - # Outside `swarm-*` and the store's own mounts it holds nothing: no - # hive's policy or role, no auth mount, no token, no secret. - name = "the granter grants nothing outside swarm-* and the store's own mounts"; + # Outside `swarm-*`, the store's own mounts and the `default` policy it + # holds nothing: no hive's policy or role, no auth mount, no token, no + # secret. + name = "the granter grants nothing outside swarm-*, the store's own mounts and the default policy"; ok = lib.all (p: grantFor granterGrants p == null) [ "sys/policies/acl/hive-x" @@ -1443,8 +1470,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all fourteen units, and each yields calls. - name = "the granter-policy check sees all fourteen granting units, and parses calls from each"; + # certificate reaches all fifteen units, and each yields calls. + name = "the granter-policy check sees all fifteen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) @@ -1706,6 +1733,77 @@ let && !(lib.elem "swarm-bao-operator-viewer-policy.service" g.after) && !(v ? postStart && lib.hasInfix "swarm-bao-granter-role" v.postStart); } + { + # Every token carries bao's `default` policy, so a read added there is + # a read for every authenticated client. Exactly one stanza, a read on + # `swarm/public/` and nothing wider; the markers are what a later run + # replaces it by. + name = "the default policy gains one read, on swarm/public/, between markers"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-public-read-policy.script; + block = lib.head ( + lib.tail ( + lib.splitString "# hyperhive: swarm/public begin\n" (builtins.unsafeDiscardStringContext s) + ) + ); + stanzas = grantsIn (lib.head (lib.splitString "# hyperhive: swarm/public end" block)); + in + stanzas == [ + { + path = "secret/data/swarm/public/*"; + caps = [ "read" ]; + } + ] + && lib.hasInfix "-v e='# hyperhive: swarm/public end'" s + && lib.hasInfix "printf '%s\\n' \"$wanted\" |\n bao policy write default -" s; + } + { + # A read-modify-write of a policy every token depends on: the read + # comes first, an empty one stops the unit before anything is written, + # and the capability probe comes before both, so a granter without the + # grant writes nothing. + name = "the default policy is read before it is written, and never written from an empty read"; + ok = + let + s = builtins.unsafeDiscardStringContext baoGrantHere.systemd.services.swarm-bao-public-read-policy.script; + at = needle: lib.stringLength (lib.head (lib.splitString needle s)); + probe = "caps=\"$(bao token capabilities sys/policies/acl/default)\""; + read = "bao read -field=policy sys/policies/acl/default"; + emptyGuard = "if [ -z \"$current\" ]; then"; + write = "bao policy write default -"; + in + lib.hasInfix probe s + && at probe < at "exit 0" + && at "exit 0" < at read + && at read < at emptyGuard + && at emptyGuard < at write + && lib.hasInfix "swarm-bao-granter-role re-runs this unit once it succeeds." s; + } + { + # The granter's own policy is what lets that unit write `default`, and a + # changed granter policy lands only with the bootstrap step, so the + # step's success restarts the unit as it does the viewer's. + name = "the granter may write the default policy, and its step restarts the public-read unit"; + ok = + let + s = baoGrantHere.systemd.services; + g = s.swarm-bao-granter-role; + p = s.swarm-bao-public-read-policy; + in + lib.any ( + gr: + gr.path == "sys/policies/acl/default" + && + gr.caps == [ + "create" + "read" + "update" + ] + ) granterGrants + && lib.hasInfix "swarm-bao-public-read-policy.service" g.serviceConfig.ExecStartPost + && lib.elem "swarm-bao-granter-role.service" p.after; + } { # The client secret is on stdin, never an argument in /proc; the rest is # the swarm's authelia and the UI's own client id.