swarm-archive: grant only once an archive is up; keep logins across a gap in placements

The archive host creates a ready marker in the store and swarm-controller
writes no credentials before it exists, so a swarm without an archive, and a
mixed-version swarm, leave every hive backup a logged no-op. A hive with no
declaration no longer counts as placing nothing: its copies and its agents'
user lines stay. The restic password is pinned by tests across a revoke, a
re-placement and a move.

nix/host-modules/hive-archive-backup.nix

@@ -12,9 +12,9 @@
 # (../../swarm-controller/src/archive_access.rs), read with the hive's own
 # store identity: the agent's login to the archive, and its restic password,
 # which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to
-# disk. An agent the controller has not granted this hive is skipped; a hive
-# with agents and no grant at all fails, which is what a swarm-controller
-# without the archive looks like.
+# disk. An agent the controller has not granted this hive is skipped. The
+# controller grants nothing until a host runs the archive, so a swarm without
+# one, and a controller without this feature, leave every run a quiet no-op.
 #
 # A failed agent prints `archive backup failed: <agent>` to the journal,
 # which the swarm's `archive backup failed` alert reads, and fails the unit
@@ -65,9 +65,9 @@ in
       default = true;
       description = ''
         Back this hive's agents up to the swarm archive. On wherever the hive
-        has its store identity; a swarm that runs no archive
-        ({option}`services.hyperhive.deploy.archive.enable` on one host) sees
-        every run fail, so turn it off there.
+        has its store identity. Where no host sets
+        {option}`services.hyperhive.deploy.archive.enable`, every run finds no
+        credentials, logs that and succeeds.
       '';
     };
 

nix/host-modules/lib/archive-backup.nix

@@ -5,7 +5,8 @@
 # same script against a stand-in `bao`.
 #
 # Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
-# and the `BAO_*` client set. Every failure prints a line starting
+# and the `BAO_*` client set. A store that cannot be reached, or an agent whose
+# backup fails, prints a line starting
 # `archive backup failed`, which the swarm's alert reads, and ends the run with
 # a non-zero status once the other agents have been tried.
 { lib }:
@@ -127,11 +128,10 @@ in
     fi
   done
 
-  # Agents here and not one of them granted: swarm-controller has not written
-  # this hive's grants, which a controller without the archive never does.
+  # No grant for any agent here: the swarm has no archive, or its controller
+  # has not granted this hive yet. Neither is a failure of this hive.
   if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
-    echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
-    failed=1
+    echo "swarm-controller has granted this hive no archive credentials ($candidates agents here); nothing to back up."
   fi
   exit "$failed"
 ''

nix/host-modules/lib/archive-server.nix

@@ -31,6 +31,7 @@ let
 
   usersPrefix = "secret/swarm/services/archive/users";
   resticPrefix = "secret/swarm/services/archive/restic";
+  readyPath = "secret/swarm/services/archive/ready";
 
   keepFlags = lib.concatStringsSep " " (
     [ "--keep-last ${toString retention.keepLast}" ]
@@ -81,6 +82,12 @@ in
       exit 1
     fi
 
+    # Tells swarm-controller an archive exists; it grants no credentials until
+    # then. Created once, never rewritten.
+    if ! store_bao kv get -field=ready ${lib.escapeShellArg readyPath} >/dev/null 2>&1; then
+      store_bao kv put -cas=0 ${lib.escapeShellArg readyPath} ready=1 >/dev/null
+    fi
+
     # A prefix that holds nothing is "No value found"; it means no agent has a
     # user yet, not a failure.
     rc=0

nix/host-modules/swarm-archive.nix

@@ -16,6 +16,8 @@
 # - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
 #   from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
 #   when it changes. The leaves hold password hashes only.
+#   The unit also creates `swarm/services/archive/ready`, which tells
+#   swarm-controller an archive exists: it grants no credentials before that.
 # - `swarm-archive-prune` (daily) reads each agent's restic password through
 #   `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
 #   therefore decrypt every backup, which is the cost of pruning where clients

nix/host-modules/swarm-bao.nix

@@ -546,6 +546,10 @@ let
       capabilities = ["list"]
     }
 
+    path "${credentialMountPath}/data/swarm/services/archive/ready" {
+      capabilities = ["read"]
+    }
+
     path "${credentialMountPath}/metadata/swarm/services/archive/users/+" {
       capabilities = ["delete"]
     }
@@ -1139,6 +1143,12 @@ let
     }
   '';
 
+  createReadStanza = path: ''
+    path "${path}" {
+      capabilities = ["create", "read"]
+    }
+  '';
+
   listStanza = path: ''
     path "${path}" {
       capabilities = ["list"]
@@ -1383,7 +1393,8 @@ let
 
   # The backup archive: the users file is rendered from one leaf per agent, and
   # the prune reads each agent's restic password. Read and list only; the
-  # controller writes both.
+  # controller writes both. It also creates the marker that tells the
+  # controller an archive exists.
   archiveReaders = [
     {
       name = "swarm-archive";
@@ -1391,7 +1402,8 @@ let
       policyText =
         readStanza "${credentialMountPath}/data/swarm/services/archive/users/*"
         + readStanza "${credentialMountPath}/data/swarm/services/archive/restic/*"
-        + listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/";
+        + listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/"
+        + createReadStanza "${credentialMountPath}/data/swarm/services/archive/ready";
     }
   ];
 

nix/module-eval/bao-grants.nix

@@ -2171,7 +2171,7 @@ let
       # The archive reads the users and the repositories' passwords and writes
       # nothing; the hive-side copies of the REST password are not its to read.
       # The controller is the only writer: the control is its own grant.
-      name = "the archive's grant is read on its users and restic passwords, list on its users, and nothing else, and only the controller writes them";
+      name = "the archive's grant is read on its users and restic passwords, list on its users, create and read on its ready marker, and nothing else, and only the controller writes them";
       ok =
         let
           s = baoGrantHere.systemd.services.swarm-bao-archive-policy.script;
@@ -2181,8 +2181,14 @@ let
         (grantFor grants "secret/data/swarm/services/archive/users/atlas").caps == [ "read" ]
         && (grantFor grants "secret/data/swarm/services/archive/restic/atlas").caps == [ "read" ]
         && (grantFor grants "secret/metadata/swarm/services/archive/users/").caps == [ "list" ]
+        &&
+          (grantFor grants "secret/data/swarm/services/archive/ready").caps == [
+            "create"
+            "read"
+          ]
+        && (grantFor c "secret/data/swarm/services/archive/ready").caps == [ "read" ]
         && grantFor grants "secret/data/swarm/hives/h1/archive/atlas" == null
-        && lib.length grants == 3
+        && lib.length grants == 4
         && lib.hasInfix "allowed_common_names=swarm-archive" s
         && lib.hasInfix "token_policies=swarm-archive" s
         && lib.elem "create" (grantFor c "secret/data/swarm/services/archive/users/atlas").caps

nix/vm-tests/swarm-archive.nix

@@ -9,7 +9,7 @@
 #   applies the retention;
 # - the users file follows the store: a user removed from it is refused without
 #   a restart of the server;
-# - a hive that has agents and no grant fails its backup.
+# - a hive that has agents and no grant backs up nothing and succeeds.
 #
 # One node, because the gateway hop is outside what is under test: the
 # backup script dials `http://127.0.0.1:<port>` instead of
@@ -33,6 +33,11 @@ let
         [ -d "$store/$path" ] || { echo "No value found at $path" >&2; exit 2; }
         ls "$store/$path" | ${pkgs.jq}/bin/jq -R . | ${pkgs.jq}/bin/jq -s .
         exit 0 ;;
+      "kv put")
+        [ -e "$store/$4" ] && { echo "check-and-set parameter did not match" >&2; exit 2; }
+        mkdir -p "$store/$4"
+        printf %s "''${5#*=}" > "$store/$4/''${5%%=*}"
+        exit 0 ;;
       "kv get")
         field="''${3#-field=}"
         [ -f "$store/$4/$field" ] || { echo "No value found at $4" >&2; exit 2; }
@@ -265,9 +270,14 @@ pkgs.testers.runNixOSTest {
         assert status("beta", "beta") == "401", status("beta", "beta")
         assert status("alpha", "alpha") == "200", status("alpha", "alpha")
 
-    with subtest("a hive with agents and no grant fails its backup"):
+    with subtest("the archive announces itself once, so swarm-controller starts granting"):
+        machine.succeed(f"test \"$(cat {store}/secret/swarm/services/archive/ready/ready)\" = 1")
+        machine.succeed("systemctl start swarm-archive-users")
+
+    with subtest("a hive with agents and no grant backs up nothing and does not fail"):
         machine.succeed(f"rm -r {store}/secret/swarm/hives/${hive}")
-        machine.fail("systemctl start test-archive-backup")
-        machine.succeed("journalctl -u test-archive-backup | grep -q 'archive backup failed: swarm-controller has granted this hive no archive credentials'")
+        machine.succeed("systemctl start test-archive-backup")
+        machine.succeed("journalctl -u test-archive-backup | grep -q 'granted this hive no archive credentials'")
+        machine.fail("journalctl -u test-archive-backup | grep -q 'archive backup failed'")
   '';
 }