swarm-archive: forget future-dated snapshots before each prune; fix the grants case

A client chooses a snapshot's time, so snapshots dated ahead could fill
--keep-last and push the real ones out of a prune. The prune now forgets
snapshots dated more than an hour after now first, and logs their ids. The
VM test forges one and checks the real snapshot stays. The bao-grants case
that read a + path as a glob now names the literal path, and two cases that
pinned 'nothing else' list the archive grants. The ciRunner stanza is back
at its original indent.

docs/2-handbook/swarm/archive.md

@@ -60,9 +60,14 @@ either rule keeps it:
 | `deploy.archive.retention.keepLast`   | `3`     | the newest this many, however old |
 
 The age rule limits what a compromised hive can do by flooding the archive with
-fake snapshots: they can't push the last three days out. Set `keepWithin` to
+fake snapshots: they can't push the last three days out. restic ranks snapshots
-`null` to leave that rule off. `deploy.archive.pruneSchedule` sets when the
+by their date, and a client chooses a snapshot's date, so a client could also
-prune runs (default `daily`).
+date snapshots years ahead to fill the `keepLast` slots. Before each prune the
+archive host forgets every snapshot dated more than an hour ahead of its own
+clock and logs `archive prune warning: <agent> has snapshots dated after now`.
+A line like that means a hive or its credentials are misbehaving. Set
+`keepWithin` to `null` to leave the age rule off. `deploy.archive.pruneSchedule`
+sets when the prune runs (default `daily`).
 
 Because the archive host prunes, it reads every agent's restic password from
 the secret store and can decrypt every backup. Treat that host as part of the

nix/host-modules/lib/archive-server.nix

@@ -33,6 +33,9 @@ let
   resticPrefix = "secret/swarm/services/archive/restic";
   readyPath = "secret/swarm/services/archive/ready";
 
+  # Slack for the clocks of the hives that stamp snapshots.
+  futureSkewSeconds = 3600;
+
   keepFlags = lib.concatStringsSep " " (
     [ "--keep-last ${toString retention.keepLast}" ]
     ++ lib.optional (
@@ -130,10 +133,37 @@ in
   # `restic forget --prune` on every repository, with its password read from
   # the store at run time. The server refuses deletes from clients, so this is
   # the only place history is dropped.
+  #
+  # restic ranks snapshots by their time, which the client chooses (`restic
+  # backup --time`). Snapshots dated years ahead would fill `--keep-last` and
+  # push the real ones out, so each prune first forgets the snapshots dated more
+  # than `futureSkewSeconds` after now. restic has no rule relative to the
+  # clock for `--keep-last`.
   pruneScript = ''
     set -euo pipefail
     ${storeSeededSecret}
 
+    # Forgets the snapshots of $repo dated after the limit, naming them in the
+    # log; returns non-zero when it cannot read the list or a time.
+    forget_future() {
+      local repo="$1" agent="$2" snaps id time at future="" limit
+      limit=$(( $(date +%s) + ${toString futureSkewSeconds} ))
+      snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json \
+        | jq -r '.[] | "\(.id) \(.time)"')" || return 1
+      while read -r id time; do
+        [ -n "$id" ] || continue
+        at="$(date -d "$time" +%s)" || return 1
+        if [ "$at" -gt "$limit" ]; then
+          future+=" $id"
+        fi
+      done <<<"$snaps"
+      if [ -n "$future" ]; then
+        echo "archive prune warning: $agent has snapshots dated after now, forgetting them:$future" >&2
+        # shellcheck disable=SC2086
+        ${restic}/bin/restic -r "$repo" --retry-lock 10m forget $future || return 1
+      fi
+    }
+
     store_init
     store_login
     if [ "$STORE_UP" != 1 ]; then
@@ -151,7 +181,8 @@ in
         continue
       fi
       export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
-      if ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
+      if forget_future "$repo" "$agent" \
+        && ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
         echo "pruned $agent"
       else
         echo "archive prune failed: $agent" >&2

nix/host-modules/swarm-bao.nix

@@ -554,7 +554,7 @@ let
       capabilities = ["delete"]
     }
 
-        path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" {
+    path "${credentialMountPath}/data/${ciRunnerTokenLeaf}" {
       capabilities = ["create", "update"]
     }
 

nix/module-eval/bao-grants.nix

@@ -1587,7 +1587,7 @@ let
       # `update` on its own role and `read` on its own two cache facts and its own
       # squid self-check credential, and each
       # hive's block names neither the other's role nor its subject.
-      name = "each hive's gateway-leaf identity may issue from its own role and read its own two cache facts and squid self-check credential, and nothing else";
+      name = "each hive's gateway-leaf identity may issue from its own role, read its own two cache facts, squid self-check credential and archive grants, and list those grants, and nothing else";
       ok =
         let
           s = baoGrantTwoHives.systemd.services.swarm-bao-hive-leaf-policy.script;
@@ -1629,6 +1629,14 @@ let
             path = "secret/data/swarm/hives/h1/squid-selfcheck";
             caps = [ "read" ];
           }
+          {
+            path = "secret/data/swarm/hives/h1/archive/*";
+            caps = [ "read" ];
+          }
+          {
+            path = "secret/metadata/swarm/hives/h1/archive/";
+            caps = [ "list" ];
+          }
           {
             path = "pki/issue/swarm-hive-leaf-h2";
             caps = [ "update" ];
@@ -1645,6 +1653,14 @@ let
             path = "secret/data/swarm/hives/h2/squid-selfcheck";
             caps = [ "read" ];
           }
+          {
+            path = "secret/data/swarm/hives/h2/archive/*";
+            caps = [ "read" ];
+          }
+          {
+            path = "secret/metadata/swarm/hives/h2/archive/";
+            caps = [ "list" ];
+          }
         ]
         && issues "h1" "h1.t.local" h1
         && issues "h2" "h2.example.org" h2
@@ -1853,7 +1869,7 @@ let
       # segment, which keeps each stanza to its one leaf; a `*` is a glob only
       # at the end of a path. Pinned as whole stanzas, so an added capability
       # fails.
-      name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, and nothing else under hives";
+      name = "the controller writes each hive's sender and forge tokens, creates its squid self-check credential, manages its archive grants, and nothing else under hives";
       ok =
         let
           s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
@@ -1862,7 +1878,12 @@ let
         && lib.hasInfix "path \"secret/data/swarm/hives/+/forge/token\" {\n  capabilities = [\"create\", \"read\", \"update\"]\n}" s
         && lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n  capabilities = [\"create\", \"read\"]\n}" s
         && !(lib.hasInfix "secret/data/swarm/hives/*" s)
-        && !(lib.hasInfix "secret/metadata/swarm/hives" s);
+        # The only metadata it reaches under hives is the archive grants: list and delete.
+        && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/\" {\n  capabilities = [\"list\"]\n}" s
+        && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n  capabilities = [\"delete\"]\n}" s
+        && lib.all (g: lib.hasInfix "/archive/" g.path) (
+          lib.filter (g: lib.hasPrefix "secret/metadata/swarm/hives" g.path) (grantsIn s)
+        );
     }
     {
       # `attic_copy` reads the cache's four facts and writes each into every
@@ -2192,9 +2213,9 @@ let
         && lib.hasInfix "allowed_common_names=swarm-archive" s
         && lib.hasInfix "token_policies=swarm-archive" s
         && lib.elem "create" (grantFor c "secret/data/swarm/services/archive/users/atlas").caps
-        && lib.elem "create" (grantFor c "secret/data/swarm/hives/h1/archive/atlas").caps
+        && lib.elem "create" (grantFor c "secret/data/swarm/hives/+/archive/*").caps
-        && (grantFor c "secret/metadata/swarm/hives/h1/archive/atlas").caps == [ "delete" ]
+        && (grantFor c "secret/metadata/swarm/hives/+/archive/+").caps == [ "delete" ]
-        && (grantFor c "secret/metadata/swarm/services/archive/users/atlas").caps == [ "delete" ]
+        && (grantFor c "secret/metadata/swarm/services/archive/users/+").caps == [ "delete" ]
         && !(lib.elem "delete" (grantFor c "secret/data/swarm/services/archive/restic/atlas").caps);
     }
     {

nix/vm-tests/swarm-archive.nix

@@ -6,7 +6,8 @@
 # - a backup of two agents lands in one repository each, and `work/` is left out;
 # - one agent's login cannot read another's repository;
 # - a client cannot delete or overwrite anything, and the server-side prune
-#   applies the retention;
+#   applies the retention, also when a client has forged a snapshot dated in
+#   the future;
 # - the users file follows the store: a user removed from it is refused without
 #   a restart of the server;
 # - a hive that has agents and no grant backs up nothing and succeeds.
@@ -146,9 +147,9 @@ pkgs.testers.runNixOSTest {
         ../host-modules/swarm-archive.nix
       ];
 
-      # Keep only the newest snapshot, so the prune is visible after two backups.
+      # The newest snapshot, plus whatever is younger than three days.
       services.hyperhive.deploy.archive.retention = {
-        keepWithin = null;
+        keepWithin = "3d";
         keepLast = 1;
       };
 
@@ -252,8 +253,8 @@ pkgs.testers.runNixOSTest {
 
     with subtest("a client cannot delete or overwrite"):
         snap = machine.succeed(restic("alpha", "alpha", "snapshots --json")).split('"id":"')[1].split('"')[0]
-        # The presence control: a lock, which a backup creates and removes, is deletable.
+        # An old snapshot for the prune to drop. The backup is also the presence control: a lock, which a backup creates and removes, is deletable.
-        machine.succeed(restic("alpha", "alpha", "backup --host alpha ${agentRoot}/alpha/state"))
+        machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2000-01-01 00:00:00' ${agentRoot}/alpha/state"))
         assert status("alpha", "alpha", f"snapshots/{snap}", "DELETE") == "403"
         assert status("alpha", "alpha", "config", "POST") == "403"
         machine.fail(restic("alpha", "alpha", "forget --keep-last 1 --prune"))
@@ -263,6 +264,19 @@ pkgs.testers.runNixOSTest {
         machine.succeed("systemctl start swarm-archive-prune")
         for agent in agents:
             assert machine.succeed(restic(agent, agent, "snapshots --json")).count('"id"') == 1
+        # The old snapshot went, the one from the hive's backup stayed.
+        assert snap in machine.succeed(restic("alpha", "alpha", "snapshots --json"))
+
+    with subtest("a snapshot dated in the future cannot push the real ones out of the retention"):
+        # restic counts --keep-within from the newest snapshot, and the client sets its time.
+        machine.succeed(restic("alpha", "alpha", "backup --host alpha --time '2100-01-01 00:00:00' ${agentRoot}/alpha/state"))
+        assert machine.succeed(restic("alpha", "alpha", "snapshots --json")).count('"id"') == 2
+        machine.succeed("systemctl start swarm-archive-prune")
+        left = machine.succeed(restic("alpha", "alpha", "snapshots --json"))
+        assert left.count('"id"') == 1 and snap in left, left
+        machine.succeed("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: alpha has snapshots dated after now'")
+        # beta's repository had nothing forged, and says nothing.
+        machine.fail("journalctl -u swarm-archive-prune | grep -q 'archive prune warning: beta'")
 
     with subtest("a user removed from the store is refused without a restart"):
         machine.succeed(f"rm -r {store}/secret/swarm/services/archive/users/beta")