From 06784e55fa25669b936fe70feb9ad5e882b33784 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 5 Oct 2026 22:13:58 +0200 Subject: [PATCH] swarm-bao: drop the default-policy grant; swarm/public/ is read by name Writing bao's `default` policy needed a new stanza in the granter's own policy, which only the bootstrap token can upload, so deploying it took a manual root-token step. Nothing needed it: the one reader of the callout xkey's public half is `swarm-bao-nats-callout-xkey`, which logs in as the `swarm-nats-auth` role, and that role's policy already names the path. `swarm-bao-nats-auth-policy` writes that policy as the granter on every deploy, under the granter's existing `sys/policies/acl/swarm-*` grant. Removed: the `swarm-bao-public-read-policy` unit, its marker block, the granter's `sys/policies/acl/default` stanza, the granter-role restart of that unit, the bao.md section and the module-eval cases pinning them. --- docs/swarm/bao.md | 21 +----- nix/host-modules/swarm-bao.nix | 119 +++------------------------------ nix/module-eval/bao-grants.nix | 107 ++++------------------------- swarm-nats-auth/src/xkey.rs | 8 +-- 4 files changed, 31 insertions(+), 224 deletions(-) diff --git a/docs/swarm/bao.md b/docs/swarm/bao.md index f5abcc61..e73c97f2 100644 --- a/docs/swarm/bao.md +++ b/docs/swarm/bao.md @@ -55,9 +55,9 @@ the unit's journal instead. ### Residual risk -The granter is root-equivalent. It may write any `swarm-*` policy, and bao's -`default` policy, with any content, and attach a policy to a role that accepts -any certificate; no bao ACL can constrain what a policy says. What bounds it: +The granter is root-equivalent. It may write any `swarm-*` policy with any +content, and attach it to a role that accepts any certificate; no bao ACL can +constrain what a policy says. What bounds it: - `nix/host-modules/swarm-bao.nix` renders every policy it writes, and module-eval pins each principal's grants. **Merging a change to that policy @@ -76,21 +76,6 @@ any certificate; no bao ACL can constrain what a policy says. What bounds it: restarting `swarm-bao-pki` mints a new leaf, but doesn't invalidate the old one. -### What every token may read - -Anything under `secret/swarm/public/` is readable by every token the store -issues. `swarm-bao-public-read-policy` puts that one read into bao's `default` -policy, which every token carries, between two marker lines. It reads the -policy, replaces only its own block, and writes the result back when it -differs, so the store's own stanzas stay as the store wrote them. It never -writes a policy it read back empty. The callout xkey's public half lives -there ([`secrets.md`](secrets.md#the-queues-callout-xkey)). - -Writing `default` is a grant on the granter's own policy, so on a store -whose granter predates it, the unit says so and exits 0 until the bootstrap -step re-runs. Until then only principals whose policy names a `swarm/public/` -path read it; the queue's responder is one, so the queue doesn't wait on this. - ## Sealing `services.hyperhive.deploy.bao.seal`: diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 5775f3eb..095e30fc 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -185,11 +185,9 @@ let # the services root. The next six set up the agent PKI mount: the mount, its # root and the `swarm-*` role agent certificates are issued through. No # `root` delete there: every agent's cert-auth role pins that root by value, - # so replacing it would lock every agent out. The next three configure the + # so replacing it would lock every agent out. The last three configure the # `oidc` auth method: its config, its `swarm-*` roles, and a read on the one - # client secret that config carries. The last is bao's `default` policy, - # which every token carries: `swarm-bao-public-read-policy` adds - # the `swarm/public/` read to it. No `sys/auth`: the auth mounts are + # client secret that config carries. No `sys/auth`: the auth mounts are # created with the bootstrap token by `swarm-bao-granter-role`. # # Piped as a shell-quoted argument like `controllerPolicyText`, so @@ -274,10 +272,6 @@ let path "${credentialMountPath}/data/${baoUiClientLeaf}" { capabilities = ["read"] } - - path "sys/policies/acl/default" { - capabilities = ["create", "read", "update"] - } ''; # What a granting unit prints when the store refuses the granter: the @@ -873,8 +867,9 @@ let ]; # The callout xkey, the nix half of `swarm-nats-auth`'s `xkey::SEED_PATH` - # and `xkey::PUBLIC_PATH`. The seed is the responder's alone; the public half - # is under `swarm/public/`, which `publicReadBlock` lets every token read. + # and `xkey::PUBLIC_PATH`. Both are readable by the `swarm-nats-auth` role + # alone: `swarm/public/` marks the half that is no secret, and no policy + # opens it to every token. xkeySeedLeaf = "swarm/services/swarm-nats/callout/xkey-seed"; xkeyPublicLeaf = "swarm/public/swarm-nats/callout-xkey"; @@ -884,20 +879,6 @@ let } ''; - # What `swarm/public/` is: readable by every token the store issues, through - # bao's `default` policy, which every token carries. A block between two - # marker lines, so `swarm-bao-public-read-policy` can replace it and leave - # the rest of that policy, the store's own, as it found it. - publicReadBegin = "# hyperhive: swarm/public begin"; - publicReadEnd = "# hyperhive: swarm/public end"; - publicReadBlock = lib.concatStringsSep "\n" [ - publicReadBegin - ''path "${credentialMountPath}/data/swarm/public/*" {'' - " capabilities = [\"read\"]" - "}" - publicReadEnd - ]; - # The CI runner's token fetch. The one reader of the path, which the # controller writes and cannot read back. ciRunnerReaders = [ @@ -2574,12 +2555,12 @@ in Restart = "on-failure"; RestartSec = 30; # Runs only once the script above exits 0, which is when the - # granter may first configure `auth/oidc` and write bao's `default` - # policy; the viewer and public-read units exit 0 without doing so - # before then, and nothing else re-runs them. A restart because - # they are `RemainAfterExit`, where a start is a no-op. + # granter may first configure `auth/oidc`; the viewer unit exits 0 + # without doing so before then, and nothing else re-runs it. A + # restart because it is `RemainAfterExit`, where a start is a + # no-op. # - # `--no-block`: both units are ordered after this one, so a + # `--no-block`: the viewer unit is ordered after this one, so a # blocking restart waits on a job that cannot run until this unit # is active, which it is not until this command returns. # @@ -2588,7 +2569,7 @@ in # already running (or a `systemctl` that itself fails to enqueue # the job) would mark the granter failed and trigger its own # `Restart=on-failure` for a problem that isn't the granter's. - ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service swarm-bao-public-read-policy.service"; + ExecStartPost = "-${pkgs.systemd}/bin/systemctl restart --no-block swarm-bao-operator-viewer-policy.service"; }; script = '' set -euo pipefail @@ -3419,84 +3400,6 @@ in ''; }; - # `publicReadBlock` in bao's `default` policy. Read, the old block - # dropped, the current one appended, and written back only when that - # differs, so the store's own stanzas stay as the store wrote them. - # - # The granter may write `default` only once `swarm-bao-granter-role` has - # run with a bootstrap token since that grant was added. Until then this - # says so and exits 0, as the viewer unit does: principals granted a - # `swarm/public/` path by name still read it, and a day of retries would - # change nothing. `swarm-bao-granter-role` restarts this unit each time - # it succeeds. - systemd.services.swarm-bao-public-read-policy = lib.mkIf haveGranter { - description = "let every bao token read swarm/public/ through the default policy"; - after = [ - "container@${cfg.machine}.service" - "swarm-bao-controller-policy.service" - ] - ++ granterAfter; - requires = [ "swarm-bao-pki.service" ]; - wantedBy = [ "multi-user.target" ]; - path = [ - baoCli - pkgs.coreutils - pkgs.gawk - ]; - environment = granterEnv; - # Same unseal wait as its siblings above, for the reason stated there. - startLimitBurst = 2880; - startLimitIntervalSec = 90000; - serviceConfig = { - Type = "oneshot"; - RemainAfterExit = true; - Restart = "on-failure"; - RestartSec = 30; - }; - script = '' - set -euo pipefail - - ${granterLogin} - - caps="$(bao token capabilities sys/policies/acl/default)" - case "$caps" in - *update*) ;; - *) - echo "the granter may not write the store's default policy yet (capabilities: $caps), so swarm/public/ is readable only where a policy names it." >&2 - echo "one-time step, as root on this host (docs/getting-started/setup.md):" >&2 - ${lib.concatMapStringsSep "\n" (l: "echo ${lib.escapeShellArg " ${l}"} >&2") granterSetupSteps} - echo "swarm-bao-granter-role re-runs this unit once it succeeds." >&2 - exit 0 - ;; - esac - - if ! current="$(bao read -field=policy sys/policies/acl/default 2>"$err")"; then - echo "could not read the store's default policy; leaving it as it is." >&2 - cat "$err" >&2 - exit 1 - fi - # Never write a default policy built from nothing: it would drop - # every stanza the store's tokens rely on for their own lookups. - if [ -z "$current" ]; then - echo "the store returned an empty default policy; leaving it as it is." >&2 - exit 1 - fi - - kept="$(printf '%s\n' "$current" | - awk -v b=${lib.escapeShellArg publicReadBegin} -v e=${lib.escapeShellArg publicReadEnd} \ - '$0 == b { skip = 1 } !skip { print } $0 == e { skip = 0 }')" - wanted="$kept - ${publicReadBlock}" - if [ "$wanted" = "$current" ]; then - echo "the default policy already lets every token read swarm/public/" - exit 0 - fi - printf '%s\n' "$wanted" | - bao policy write default - - echo "the default policy now lets every token read swarm/public/" - ''; - }; - # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index 5608ba8a..51ce3b2c 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -170,8 +170,8 @@ let _: u: (u.environment.BAO_CLIENT_CERT or null) == granterCertFile ) baoGrantWithConsumers.systemd.services; - # The fifteen units that write a grant as the granter, by name, for the - # discovery control below. + # The fourteen units that write a `swarm-*` grant, by name, for the discovery + # control below. grantingUnitNames = [ "swarm-bao-controller-policy" "swarm-bao-secret-publisher-policy" @@ -187,7 +187,6 @@ let "swarm-bao-operator-viewer-policy" "swarm-bao-hive-leaf-policy" "swarm-bao-ci-runner-policy" - "swarm-bao-public-read-policy" ]; # Comment lines dropped first: both the HCL and the scripts explain @@ -813,24 +812,24 @@ let } { # A store host without the granter's pair writes its grants some other - # way, so none of the fifteen units may exist. Without this arm + # way, so none of the fourteen units may exist. Without this arm # `lib.mkIf haveGranter` could be dropped from any of them and every other # case here would still pass. - name = "without the granter's pair none of the fifteen granting units render"; + name = "without the granter's pair none of the fourteen granting units render"; ok = let s = baoGranterOptOut.systemd.services; in lib.all (unit: !(s ? ${unit})) (grantingUnitNames ++ [ "swarm-bao-granter-role" ]) - # The control: the same store with the pair renders all fifteen. + # The control: the same store with the pair renders all fourteen. && lib.all (unit: baoGrantHere.systemd.services ? ${unit}) grantingUnitNames; } { - # 🩸 What replaced the silent skip. With no bootstrap token the fifteen still + # 🩸 What replaced the silent skip. With no bootstrap token the fourteen still # render, and a refused granter fails them with the step that fixes it. # A store host that never named a token is told to name one, since the # unit that sets the granter up renders only where it has. - name = "a store host without a bootstrap token renders the fifteen, each failing loudly with the one-time step"; + name = "a store host without a bootstrap token renders the fourteen, each failing loudly with the one-time step"; ok = let s = baoGranterNoToken.systemd.services; @@ -937,7 +936,7 @@ let { # The granter's grants, whole. Pinned as the full list, because an added # path or capability is exactly what a presence check misses. - name = "the granter's policy is exactly these twenty-one stanzas"; + name = "the granter's policy is exactly these twenty stanzas"; ok = let cu = [ @@ -1029,14 +1028,6 @@ let path = "secret/data/swarm/services/swarm-bao-ui/oidc/client"; caps = [ "read" ]; } - { - path = "sys/policies/acl/default"; - caps = [ - "create" - "read" - "update" - ]; - } ]; } { @@ -1051,10 +1042,9 @@ let ]; } { - # Outside `swarm-*`, the store's own mounts and the `default` policy it - # holds nothing: no hive's policy or role, no auth mount, no token, no - # secret. - name = "the granter grants nothing outside swarm-*, the store's own mounts and the default policy"; + # Outside `swarm-*` and the store's own mounts it holds nothing: no + # hive's policy or role, no auth mount, no token, no secret. + name = "the granter grants nothing outside swarm-* and the store's own mounts"; ok = lib.all (p: grantFor granterGrants p == null) [ "sys/policies/acl/hive-x" @@ -1470,8 +1460,8 @@ let } { # What makes the case above mean something: discovery by the granter's - # certificate reaches all fifteen units, and each yields calls. - name = "the granter-policy check sees all fifteen granting units, and parses calls from each"; + # certificate reaches all fourteen units, and each yields calls. + name = "the granter-policy check sees all fourteen granting units, and parses calls from each"; ok = lib.sort lib.lessThan (lib.attrNames granterUnits) == lib.sort lib.lessThan grantingUnitNames && lib.all (u: baoCalls u.script != [ ]) (lib.attrValues granterUnits) @@ -1733,77 +1723,6 @@ let && !(lib.elem "swarm-bao-operator-viewer-policy.service" g.after) && !(v ? postStart && lib.hasInfix "swarm-bao-granter-role" v.postStart); } - { - # Every token carries bao's `default` policy, so a read added there is - # a read for every authenticated client. Exactly one stanza, a read on - # `swarm/public/` and nothing wider; the markers are what a later run - # replaces it by. - name = "the default policy gains one read, on swarm/public/, between markers"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-public-read-policy.script; - block = lib.head ( - lib.tail ( - lib.splitString "# hyperhive: swarm/public begin\n" (builtins.unsafeDiscardStringContext s) - ) - ); - stanzas = grantsIn (lib.head (lib.splitString "# hyperhive: swarm/public end" block)); - in - stanzas == [ - { - path = "secret/data/swarm/public/*"; - caps = [ "read" ]; - } - ] - && lib.hasInfix "-v e='# hyperhive: swarm/public end'" s - && lib.hasInfix "printf '%s\\n' \"$wanted\" |\n bao policy write default -" s; - } - { - # A read-modify-write of a policy every token depends on: the read - # comes first, an empty one stops the unit before anything is written, - # and the capability probe comes before both, so a granter without the - # grant writes nothing. - name = "the default policy is read before it is written, and never written from an empty read"; - ok = - let - s = builtins.unsafeDiscardStringContext baoGrantHere.systemd.services.swarm-bao-public-read-policy.script; - at = needle: lib.stringLength (lib.head (lib.splitString needle s)); - probe = "caps=\"$(bao token capabilities sys/policies/acl/default)\""; - read = "bao read -field=policy sys/policies/acl/default"; - emptyGuard = "if [ -z \"$current\" ]; then"; - write = "bao policy write default -"; - in - lib.hasInfix probe s - && at probe < at "exit 0" - && at "exit 0" < at read - && at read < at emptyGuard - && at emptyGuard < at write - && lib.hasInfix "swarm-bao-granter-role re-runs this unit once it succeeds." s; - } - { - # The granter's own policy is what lets that unit write `default`, and a - # changed granter policy lands only with the bootstrap step, so the - # step's success restarts the unit as it does the viewer's. - name = "the granter may write the default policy, and its step restarts the public-read unit"; - ok = - let - s = baoGrantHere.systemd.services; - g = s.swarm-bao-granter-role; - p = s.swarm-bao-public-read-policy; - in - lib.any ( - gr: - gr.path == "sys/policies/acl/default" - && - gr.caps == [ - "create" - "read" - "update" - ] - ) granterGrants - && lib.hasInfix "swarm-bao-public-read-policy.service" g.serviceConfig.ExecStartPost - && lib.elem "swarm-bao-granter-role.service" p.after; - } { # The client secret is on stdin, never an argument in /proc; the rest is # the swarm's authelia and the UI's own client id. diff --git a/swarm-nats-auth/src/xkey.rs b/swarm-nats-auth/src/xkey.rs index e6859bb8..93eb8ff1 100644 --- a/swarm-nats-auth/src/xkey.rs +++ b/swarm-nats-auth/src/xkey.rs @@ -1,10 +1,10 @@ //! The callout xkey: the curve keypair `nats-server` seals every auth request //! to, and that this responder seals every reply back with. //! -//! Both halves live in the secret store. The seed is at [`SEED_PATH`], which -//! only the `swarm-nats-auth` role reads. The public half is at -//! [`PUBLIC_PATH`], under `swarm/public/`, which every token the store issues -//! may read; the queue's host renders it into `auth_callout.xkey` from there. +//! Both halves live in the secret store, and only the `swarm-nats-auth` role +//! reads either. The seed is at [`SEED_PATH`]; the public half is at +//! [`PUBLIC_PATH`], and the queue's host renders it into `auth_callout.xkey` +//! from there. //! //! [`ensure`] creates the pair when the seed is absent and otherwise only //! reads it. [`load`] only ever reads, and is what the responder uses: a