swarm-archive: grant only once an archive is up; keep logins across a gap in placements

The archive host creates a ready marker in the store and swarm-controller
writes no credentials before it exists, so a swarm without an archive, and a
mixed-version swarm, leave every hive backup a logged no-op. A hive with no
declaration no longer counts as placing nothing: its copies and its agents'
user lines stay. The restic password is pinned by tests across a revoke, a
re-placement and a move.

nix/host-modules/hive-archive-backup.nix

@@ -12,9 +12,9 @@
 # (../../swarm-controller/src/archive_access.rs), read with the hive's own
 # store identity: the agent's login to the archive, and its restic password,
 # which restic reads through `RESTIC_PASSWORD_COMMAND` and nothing writes to
-# disk. An agent the controller has not granted this hive is skipped; a hive
-# with agents and no grant at all fails, which is what a swarm-controller
-# without the archive looks like.
+# disk. An agent the controller has not granted this hive is skipped. The
+# controller grants nothing until a host runs the archive, so a swarm without
+# one, and a controller without this feature, leave every run a quiet no-op.
 #
 # A failed agent prints `archive backup failed: <agent>` to the journal,
 # which the swarm's `archive backup failed` alert reads, and fails the unit
@@ -65,9 +65,9 @@ in
       default = true;
       description = ''
         Back this hive's agents up to the swarm archive. On wherever the hive
-        has its store identity; a swarm that runs no archive
-        ({option}`services.hyperhive.deploy.archive.enable` on one host) sees
-        every run fail, so turn it off there.
+        has its store identity. Where no host sets
+        {option}`services.hyperhive.deploy.archive.enable`, every run finds no
+        credentials, logs that and succeeds.
       '';
     };
 

nix/host-modules/lib/archive-backup.nix

@@ -5,7 +5,8 @@
 # same script against a stand-in `bao`.
 #
 # Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
-# and the `BAO_*` client set. Every failure prints a line starting
+# and the `BAO_*` client set. A store that cannot be reached, or an agent whose
+# backup fails, prints a line starting
 # `archive backup failed`, which the swarm's alert reads, and ends the run with
 # a non-zero status once the other agents have been tried.
 { lib }:
@@ -127,11 +128,10 @@ in
     fi
   done
 
-  # Agents here and not one of them granted: swarm-controller has not written
-  # this hive's grants, which a controller without the archive never does.
+  # No grant for any agent here: the swarm has no archive, or its controller
+  # has not granted this hive yet. Neither is a failure of this hive.
   if [ "$candidates" -gt 0 ] && [ -z "$granted" ]; then
-    echo "archive backup failed: swarm-controller has granted this hive no archive credentials ($candidates agents here)" >&2
-    failed=1
+    echo "swarm-controller has granted this hive no archive credentials ($candidates agents here); nothing to back up."
   fi
   exit "$failed"
 ''

nix/host-modules/lib/archive-server.nix

@@ -31,6 +31,7 @@ let
 
   usersPrefix = "secret/swarm/services/archive/users";
   resticPrefix = "secret/swarm/services/archive/restic";
+  readyPath = "secret/swarm/services/archive/ready";
 
   keepFlags = lib.concatStringsSep " " (
     [ "--keep-last ${toString retention.keepLast}" ]
@@ -81,6 +82,12 @@ in
       exit 1
     fi
 
+    # Tells swarm-controller an archive exists; it grants no credentials until
+    # then. Created once, never rewritten.
+    if ! store_bao kv get -field=ready ${lib.escapeShellArg readyPath} >/dev/null 2>&1; then
+      store_bao kv put -cas=0 ${lib.escapeShellArg readyPath} ready=1 >/dev/null
+    fi
+
     # A prefix that holds nothing is "No value found"; it means no agent has a
     # user yet, not a failure.
     rc=0

nix/host-modules/swarm-archive.nix

@@ -16,6 +16,8 @@
 # - `swarm-archive-users` (root, every two minutes) renders the htpasswd file
 #   from the users leaves into `/run`, which is tmpfs. rclone re-reads the file
 #   when it changes. The leaves hold password hashes only.
+#   The unit also creates `swarm/services/archive/ready`, which tells
+#   swarm-controller an archive exists: it grants no credentials before that.
 # - `swarm-archive-prune` (daily) reads each agent's restic password through
 #   `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
 #   therefore decrypt every backup, which is the cost of pruning where clients

nix/host-modules/swarm-bao.nix

@@ -546,6 +546,10 @@ let
       capabilities = ["list"]
     }
 
+    path "${credentialMountPath}/data/swarm/services/archive/ready" {
+      capabilities = ["read"]
+    }
+
     path "${credentialMountPath}/metadata/swarm/services/archive/users/+" {
       capabilities = ["delete"]
     }
@@ -1139,6 +1143,12 @@ let
     }
   '';
 
+  createReadStanza = path: ''
+    path "${path}" {
+      capabilities = ["create", "read"]
+    }
+  '';
+
   listStanza = path: ''
     path "${path}" {
       capabilities = ["list"]
@@ -1383,7 +1393,8 @@ let
 
   # The backup archive: the users file is rendered from one leaf per agent, and
   # the prune reads each agent's restic password. Read and list only; the
-  # controller writes both.
+  # controller writes both. It also creates the marker that tells the
+  # controller an archive exists.
   archiveReaders = [
     {
       name = "swarm-archive";
@@ -1391,7 +1402,8 @@ let
       policyText =
         readStanza "${credentialMountPath}/data/swarm/services/archive/users/*"
         + readStanza "${credentialMountPath}/data/swarm/services/archive/restic/*"
-        + listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/";
+        + listStanza "${credentialMountPath}/metadata/swarm/services/archive/users/"
+        + createReadStanza "${credentialMountPath}/data/swarm/services/archive/ready";
     }
   ];