swarm-archive: check each agent's restic repository weekly

A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.

nix/host-modules/lib/archive-server.nix

@@ -1,6 +1,6 @@
 # The archive server's moving parts: the rclone command that serves the
 # repositories, the script that renders its users file from the secret store,
-# and the script that prunes each repository. A pure function, NOT a NixOS
+# the script that prunes each repository and the one that checks them. A pure function, NOT a NixOS
 # module: ../swarm-archive.nix wraps the pieces in units, and the VM test
 # (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
 # `bao`.
@@ -33,6 +33,9 @@
   retention,
   # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
   otlpEndpoint ? null,
+  # `restic check --read-data-subset=<this>`: what share of each repository's data the
+  # check reads, e.g. `5%`.
+  checkSubset ? "5%",
 }:
 let
   atomicWriteSecret = import ./atomic-write-secret.nix { };
@@ -46,6 +49,16 @@ let
   s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
   s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
 
+  # Takes the lock the prune and the check share, when the unit sets
+  # `ARCHIVE_MAINTENANCE_LOCK` (a path in a directory both units can write); waits
+  # for the holder. Held until the script exits. Needs `flock` on the unit's `path`.
+  maintenanceLock = ''
+    if [ -n "''${ARCHIVE_MAINTENANCE_LOCK:-}" ]; then
+      exec 9>"$ARCHIVE_MAINTENANCE_LOCK"
+      flock 9
+    fi
+  '';
+
   # Slack for the clocks of the hives that stamp snapshots.
   futureSkewSeconds = 3600;
 
@@ -223,6 +236,7 @@ in
   pruneScript = ''
     set -euo pipefail
     ${storeSeededSecret}
+    ${maintenanceLock}
 
     # Forgets the snapshots of $repo dated after the limit, naming them in the
     # log; returns non-zero when it cannot read the list or a time.
@@ -311,4 +325,70 @@ in
     done
     exit "$failed"
   '';
+
+  # `restic check` on every repository, reading `checkSubset` of its data: a random
+  # share each run, so repeated runs cover the repository without any run
+  # reading all of it. The repository's structure (index, snapshots, trees) is
+  # always checked in full. One repository's failure does not stop the others;
+  # the run exits non-zero if any failed.
+  checkScript = ''
+    set -euo pipefail
+    ${storeSeededSecret}
+    ${maintenanceLock}
+
+    ${otlpGauges}
+    # Pushes the outcome of one repository's check. Never fails the run.
+    push_check_metrics() {
+      local agent="$1" ok="$2" start="$3" end="$4"
+      otlp_gauges swarm-archive-check "agent=$agent" \
+        "hyperhive.archive.check.last_run_success=$ok" \
+        "hyperhive.archive.check.last_run_timestamp_seconds=$end" \
+        "hyperhive.archive.check.duration_seconds=$(( end - start ))"
+    }
+
+    # Checks the repository of each agent named in the arguments; returns
+    # non-zero if any check failed.
+    check_agents() {
+      local agent repo start ok failed=0
+      for agent in "$@"; do
+        if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
+          echo "archive check failed: $agent is not an agent name" >&2
+          failed=1
+          continue
+        fi
+        repo=${repoOf}
+        export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
+        ok=0
+        start="$(date +%s)"
+        if ${restic}/bin/restic -r "$repo" --retry-lock 10m check --read-data-subset=${lib.escapeShellArg checkSubset}; then
+          echo "checked $agent"
+          ok=1
+        else
+          echo "archive check failed: $agent" >&2
+          failed=1
+        fi
+        push_check_metrics "$agent" "$ok" "$start" "$(date +%s)"
+      done
+      return "$failed"
+    }
+
+    store_init
+    store_login
+    if [ "$STORE_UP" != 1 ]; then
+      echo "archive check failed: could not log in to the swarm secret store" >&2
+      exit 1
+    fi
+
+    ${lib.optionalString (s3 != null) s3Credentials}
+    list_agents() {
+      ${listAgents}
+    }
+    # Under `set -e` a failed listing ends the run; inside the `for` it would not.
+    agents="$(list_agents)"
+
+    rc=0
+    # shellcheck disable=SC2086
+    check_agents $agents || rc=$?
+    exit "$rc"
+  '';
 }