From 793d987a883286d38a3dd4eebb7207277cce277d Mon Sep 17 00:00:00 2001 From: atlas Date: Tue, 10 Nov 2026 08:32:54 +0100 Subject: [PATCH] swarm-archive: forget future-dated snapshots before each prune; fix the grants case A client chooses a snapshot's time, so snapshots dated ahead could fill --keep-last and push the real ones out of a prune. The prune now forgets snapshots dated more than an hour after now first, and logs their ids. The VM test forges one and checks the real snapshot stays. The bao-grants case that read a + path as a glob now names the literal path, and two cases that pinned 'nothing else' list the archive grants. The ciRunner stanza is back at its original indent. --- docs/2-handbook/swarm/archive.md | 11 ++++++--- nix/host-modules/lib/archive-server.nix | 33 ++++++++++++++++++++++++- nix/host-modules/swarm-bao.nix | 2 +- nix/module-eval/bao-grants.nix | 33 ++++++++++++++++++++----- nix/vm-tests/swarm-archive.nix | 24 ++++++++++++++---- 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/docs/2-handbook/swarm/archive.md b/docs/2-handbook/swarm/archive.md index b31c28c49..72779b38b 100644 --- a/docs/2-handbook/swarm/archive.md +++ b/docs/2-handbook/swarm/archive.md @@ -60,9 +60,14 @@ either rule keeps it: | `deploy.archive.retention.keepLast` | `3` | the newest this many, however old | The age rule limits what a compromised hive can do by flooding the archive with -fake snapshots: they can't push the last three days out. Set `keepWithin` to -`null` to leave that rule off. `deploy.archive.pruneSchedule` sets when the -prune runs (default `daily`). +fake snapshots: they can't push the last three days out. restic ranks snapshots +by their date, and a client chooses a snapshot's date, so a client could also +date snapshots years ahead to fill the `keepLast` slots. Before each prune the +archive host forgets every snapshot dated more than an hour ahead of its own +clock and logs `archive prune warning: has snapshots dated after now`. +A line like that means a hive or its credentials are misbehaving. Set +`keepWithin` to `null` to leave the age rule off. `deploy.archive.pruneSchedule` +sets when the prune runs (default `daily`). Because the archive host prunes, it reads every agent's restic password from the secret store and can decrypt every backup. Treat that host as part of the diff --git a/nix/host-modules/lib/archive-server.nix b/nix/host-modules/lib/archive-server.nix index c76082242..a052bc747 100644 --- a/nix/host-modules/lib/archive-server.nix +++ b/nix/host-modules/lib/archive-server.nix @@ -33,6 +33,9 @@ let resticPrefix = "secret/swarm/services/archive/restic"; readyPath = "secret/swarm/services/archive/ready"; + # Slack for the clocks of the hives that stamp snapshots. + futureSkewSeconds = 3600; + keepFlags = lib.concatStringsSep " " ( [ "--keep-last ${toString retention.keepLast}" ] ++ lib.optional ( @@ -130,10 +133,37 @@ in # `restic forget --prune` on every repository, with its password read from # the store at run time. The server refuses deletes from clients, so this is # the only place history is dropped. + # + # restic ranks snapshots by their time, which the client chooses (`restic + # backup --time`). Snapshots dated years ahead would fill `--keep-last` and + # push the real ones out, so each prune first forgets the snapshots dated more + # than `futureSkewSeconds` after now. restic has no rule relative to the + # clock for `--keep-last`. pruneScript = '' set -euo pipefail ${storeSeededSecret} + # Forgets the snapshots of $repo dated after the limit, naming them in the + # log; returns non-zero when it cannot read the list or a time. + forget_future() { + local repo="$1" agent="$2" snaps id time at future="" limit + limit=$(( $(date +%s) + ${toString futureSkewSeconds} )) + snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json \ + | jq -r '.[] | "\(.id) \(.time)"')" || return 1 + while read -r id time; do + [ -n "$id" ] || continue + at="$(date -d "$time" +%s)" || return 1 + if [ "$at" -gt "$limit" ]; then + future+=" $id" + fi + done <<<"$snaps" + if [ -n "$future" ]; then + echo "archive prune warning: $agent has snapshots dated after now, forgetting them:$future" >&2 + # shellcheck disable=SC2086 + ${restic}/bin/restic -r "$repo" --retry-lock 10m forget $future || return 1 + fi + } + store_init store_login if [ "$STORE_UP" != 1 ]; then @@ -151,7 +181,8 @@ in continue fi export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent" - if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then + if forget_future "$repo" "$agent" \ + && ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then echo "pruned $agent" else echo "archive prune failed: $agent" >&2 diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index bd01dda99..6d226a3d1 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -554,7 +554,7 @@ let capabilities = ["delete"] } - path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" { + path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" { capabilities = ["create", "update"] } diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index c8395f348..69598b81e 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1587,7 +1587,7 @@ let # `update` on its own role and `read` on its own two cache facts and its own # squid self-check credential, and each # hive's block names neither the other's role nor its subject. - name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts and squid self-check credential, and nothing else"; + name = "each hive's gateway-leaf identity may issue from its own role, read its own two cache facts, squid self-check credential and archive grants, and list those grants, and nothing else"; ok = let s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script; @@ -1629,6 +1629,14 @@ let path = "secret/data/swarm/hives/h1/squid-selfcheck"; caps = [ "read" ]; } + { + path = "secret/data/swarm/hives/h1/archive/*"; + caps = [ "read" ]; + } + { + path = "secret/metadata/swarm/hives/h1/archive/"; + caps = [ "list" ]; + } { path = "pki/issue/swarm-hive-leaf-h2"; caps = [ "update" ]; @@ -1645,6 +1653,14 @@ let path = "secret/data/swarm/hives/h2/squid-selfcheck"; caps = [ "read" ]; } + { + path = "secret/data/swarm/hives/h2/archive/*"; + caps = [ "read" ]; + } + { + path = "secret/metadata/swarm/hives/h2/archive/"; + caps = [ "list" ]; + } ] && issues "h1" "h1.t.local" h1 && issues "h2" "h2.example.org" h2 @@ -1853,7 +1869,7 @@ let # segment, which keeps each stanza to its one leaf; a `*` is a glob only # at the end of a path. Pinned as whole stanzas, so an added capability # fails. - name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives"; + name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, manages its archive grants, and nothing else under hives"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; @@ -1862,7 +1878,12 @@ let && lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n capabilities = [\"create\", \"read\", \"update\"]\n}" s && lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s && !(lib.hasInfix "secret/data/swarm/hives/*" s) - && !(lib.hasInfix "secret/metadata/swarm/hives" s); + # The only metadata it reaches under hives is the archive grants: list and delete. + && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/\" {\n capabilities = [\"list\"]\n}" s + && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s + && lib.all (g: lib.hasInfix "/archive/" g.path) ( + lib.filter (g: lib.hasPrefix "secret/metadata/swarm/hives" g.path) (grantsIn s) + ); } { # `attic_copy` reads the cache's four facts and writes each into every @@ -2192,9 +2213,9 @@ let && lib.hasInfix "allowed_common_names=swarm-archive" s && lib.hasInfix "token_policies=swarm-archive" s && lib.elem "create" (grantFor c "secret/data/swarm/services/archive/users/atlas").caps - && lib.elem "create" (grantFor c "secret/data/swarm/hives/h1/archive/atlas").caps - && (grantFor c "secret/metadata/swarm/hives/h1/archive/atlas").caps == [ "delete" ] - && (grantFor c "secret/metadata/swarm/services/archive/users/atlas").caps == [ "delete" ] + && lib.elem "create" (grantFor c "secret/data/swarm/hives/+/archive/*").caps + && (grantFor c "secret/metadata/swarm/hives/+/archive/+").caps == [ "delete" ] + && (grantFor c "secret/metadata/swarm/services/archive/users/+").caps == [ "delete" ] && !(lib.elem "delete" (grantFor c "secret/data/swarm/services/archive/restic/atlas").caps); } { diff --git a/nix/vm-tests/swarm-archive.nix b/nix/vm-tests/swarm-archive.nix index 90c6ae77b..90858b699 100644 --- a/nix/vm-tests/swarm-archive.nix +++ b/nix/vm-tests/swarm-archive.nix @@ -6,7 +6,8 @@ # - a backup of two agents lands in one repository each, and `work/` is left out; # - one agent's login cannot read another's repository; # - a client cannot delete or overwrite anything, and the server-side prune -# applies the retention; +# applies the retention, also when a client has forged a snapshot dated in +# the future; # - the users file follows the store: a user removed from it is refused without # a restart of the server; # - a hive that has agents and no grant backs up nothing and succeeds. @@ -146,9 +147,9 @@ pkgs.testers.runNixOSTest { ../host-modules/swarm-archive.nix ]; - # Keep only the newest snapshot, so the prune is visible after two backups. + # The newest snapshot, plus whatever is younger than three days. services.hyperhive.deploy.archive.retention = { - keepWithin = null; + keepWithin = "3d"; keepLast = 1; }; @@ -252,8 +253,8 @@ pkgs.testers.runNixOSTest { with subtest("a client cannot delete or overwrite"): snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0] - # The presence control: a lock, which a backup creates and removes, is deletable. - machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state")) + # An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable. + machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state")) assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403" assert status("alpha", "alpha", "config", "POST") == "403" machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune")) @@ -263,6 +264,19 @@ pkgs.testers.runNixOSTest { machine.succeed("systemctl start swarm-archive-prune") for agent in agents: assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1 + # The old snapshot went, the one from the hive's backup stayed. + assert snap in machine.succeed(restic("alpha", "alpha", "snapshots --json")) + + with subtest("a snapshot dated in the future cannot push the real ones out of the retention"): + # restic counts --keep-within from the newest snapshot, and the client sets its time. + machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state")) + assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2 + machine.succeed("systemctl start swarm-archive-prune") + left = machine.succeed(restic("alpha", "alpha", "snapshots --json")) + assert left.count('"id"') == 1 and snap in left, left + machine.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'") + # beta's repository had nothing forged, and says nothing. + machine.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'") with subtest("a user removed from the store is refused without a restart"): machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")