archive: push restic backup and prune metrics, and add a backups dashboard

docs/2-handbook/swarm/archive.md

@@ -134,6 +134,8 @@ On a hive, `systemctl status hive-archive-backup` shows the last run, and
 `journalctl -u hive-archive-backup` lists every agent it backed up or failed to
 back up. `systemctl list-timers hive-archive-backup` shows the next run.
 
+The Grafana board `hyperhive · backups (restic)` shows the same per agent: the age of the last success, the last result, bytes added and snapshots kept. See [telemetry](../../4-implementation/swarm/telemetry.md).
+
 To see an agent's snapshots, ask the archive with that agent's password, which
 the secret store holds at `swarm/hives/<hive>/archive/<agent>` (fields
 `rest-password` and `restic-password`):

docs/4-implementation/swarm/telemetry.md

@@ -55,6 +55,7 @@ kept.
 
 - `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. It also has the tools called, the commands bash tasks ran, reminders per hour, the first-turn context size and fresh sessions per agent: everything the `/stats` page shows, except the per-bucket token bars the **cost** tab covers. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds.
 - `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent.
+- `archive`: the restic backups of every agent to the swarm archive. Per agent: time since the last successful backup (red over two hours, twice the default hourly schedule), the last run's result, duration and bytes added, then snapshots, repository size and the last prune. Failures and recent runs come from the `hive-archive-backup` and `swarm-archive-prune` logs. The numbers are the `hyperhive.archive.backup.*` gauges, which each hive's backup unit pushes after every agent, and the `hyperhive.archive.prune.*` and `hyperhive.archive.repo.*` gauges, which the prune unit pushes after every repository when its host runs a hive. A hive that predates them shows no data for its agents; the log panels work regardless. Filter by hive and agent.
 - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue.
 - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents.
 - `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, the busiest agents and destinations, and data transferred by domain over time (the top 10 domains plus an other line, summed from the proxy's access log in VictoriaLogs; the proxy logs a CONNECT tunnel when it closes, so a long-lived connection shows as one spike at close). **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent and by domain, to see which agents queried a domain and when.

nix/checks.nix

@@ -80,6 +80,9 @@ in
   # Runs ./host-modules/lib/database-size-push.nix against a symlinked directory,
   # as a dynamic user's state directory is: the reported size is the target's.
   script-test-database-size-push = import ./script-tests/database-size-push.nix { inherit pkgs; };
+  # Runs ./host-modules/lib/otlp-gauges.nix's `otlp_gauges` against a stub `curl`: the
+  # payload shape, a failed push and a bad value both return 0, no endpoint pushes nothing.
+  script-test-otlp-gauges = import ./script-tests/otlp-gauges.nix { inherit pkgs; };
   # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent
   # trust-bundle fetch retries through inside its start job: success after two
   # failures, the wait schedule, and a return of 1 once the budget is spent.

nix/host-modules/hive-archive-backup.nix

@@ -19,6 +19,9 @@
 # A failed agent prints `archive backup failed: <agent>` to the journal,
 # which the swarm's `archive backup failed` alert reads, and fails the unit
 # after the others have been tried.
+#
+# Each agent's run also pushes `hyperhive.archive.backup.*` gauges to this
+# hive's collector (./lib/archive-backup.nix); a failed push is only a warning.
 {
   pkgs,
   lib,
@@ -56,6 +59,7 @@ let
     agentRoot = "/var/lib/hyperhive/agents";
     url = "https://${hyperhiveCfg.swarm.archive.ownDomain}";
     inherit caBundle;
+    otlpEndpoint = "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
   };
 in
 {
@@ -137,6 +141,7 @@ in
         baoDeploy.package
         pkgs.btrfs-progs
         pkgs.coreutils
+        pkgs.curl
         pkgs.gnugrep
         pkgs.jq
       ];

nix/host-modules/lib/archive-backup.nix

@@ -4,11 +4,15 @@
 # wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
 # same script against a stand-in `bao`.
 #
-# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
+# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
 # and the `BAO_*` client set. A store that cannot be reached, or an agent whose
 # backup fails, prints a line starting
 # `archive backup failed`, which the swarm's alert reads, and ends the run with
 # a non-zero status once the other agents have been tried.
+#
+# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute)
+# through ./otlp-gauges.nix: last run and last success time, result, wall time,
+# bytes added and files new and changed. A failed agent still reports its result.
 { lib }:
 {
   restic,
@@ -25,11 +29,15 @@
   excludes,
   # A CA file to trust instead of restic's default, or null.
   caBundle,
+  # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
+  otlpEndpoint ? null,
 }:
 let
   storeSeededSecret = import ./store-seeded-secret.nix { };
 
   grants = hive: "secret/swarm/hives/${hive}/archive";
+
+  otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
 in
 ''
   set -euo pipefail
@@ -45,6 +53,7 @@ in
   ${lib.optionalString (caBundle != null) ''
     export RESTIC_CACERT=${lib.escapeShellArg caBundle}
   ''}
+  ${otlpGauges}
   root=${lib.escapeShellArg agentRoot}
   grants=${lib.escapeShellArg (grants hive)}
 
@@ -83,9 +92,14 @@ in
     if [ "''${#targets[@]}" = 0 ]; then
       echo "$agent has none of its backup paths yet; nothing to back up."
     else
-      ${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
+      # `--json` for the summary line the metrics read. Only that line and the
+      # errors are printed; the progress lines are dropped.
+      ${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
         ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
-        -- "''${targets[@]}" || rc=$?
+        -- "''${targets[@]}" |
+        jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
+          | if $j == null or $j.message_type == "error" or $j.message_type == "summary"
+            then $line else empty end' || rc=$?
     fi
     if [ -n "$snap" ]; then
       btrfs subvolume delete "$snap" >/dev/null || rc=1
@@ -93,6 +107,29 @@ in
     return "$rc"
   }
 
+  # One agent's run as gauges: `$5` is its output, whose `summary` line has the
+  # counts when restic got that far. Never fails the run.
+  push_backup_metrics() {
+    local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary
+    local metrics=(
+      "hyperhive.archive.backup.last_run_timestamp_seconds=$finished"
+      "hyperhive.archive.backup.last_run_success=$ok"
+      "hyperhive.archive.backup.duration_seconds=$((finished - started))"
+    )
+    if [ "$ok" = 1 ]; then
+      metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
+      summary="$(printf '%s\n' "$out" | jq -R -r '
+        fromjson? | select(type == "object" and .message_type == "summary")
+        | "hyperhive.archive.backup.data_added_bytes=\(.data_added)",
+          "hyperhive.archive.backup.files_new=\(.files_new)",
+          "hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary=""
+      while IFS= read -r line; do
+        [ -z "$line" ] || metrics+=("$line")
+      done <<<"$summary"
+    fi
+    otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}"
+  }
+
   # The agents the controller granted this hive; none yet is "No value found".
   rc=0
   granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
@@ -119,13 +156,19 @@ in
       echo "$agent has no archive credentials on this hive; skipping."
       continue
     fi
-    if (backup_agent "$agent"); then
+    started="$(date +%s)"
+    ok=0
+    out="$(backup_agent "$agent")" && ok=1
+    [ -z "$out" ] || printf '%s\n' "$out"
+    finished="$(date +%s)"
+    if [ "$ok" = 1 ]; then
       backed_up=$((backed_up + 1))
       echo "backed up $agent"
     else
       echo "archive backup failed: $agent" >&2
       failed=1
     fi
+    push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out"
   done
 
   # No grant for any agent here: the swarm has no archive, or its controller

nix/host-modules/lib/archive-server.nix

@@ -31,11 +31,14 @@
   authGroup,
   # `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
   retention,
+  # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
+  otlpEndpoint ? null,
 }:
 let
   atomicWriteSecret = import ./atomic-write-secret.nix { };
   refreshConsumer = import ./refresh-consumer.nix { };
   storeSeededSecret = import ./store-seeded-secret.nix { };
+  otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
 
   usersPrefix = "secret/swarm/services/archive/users";
   resticPrefix = "secret/swarm/services/archive/restic";
@@ -242,6 +245,36 @@ in
       fi
     }
 
+    ${otlpGauges}
+    # Pushes one repository's gauges: the outcome of this prune, the snapshot
+    # count and newest snapshot time, and the size of the data in the repository.
+    # Reads the repository after the prune. Never fails the run.
+    push_repo_metrics() {
+      local repo="$1" agent="$2" ok="$3" snaps stats time at latest="" count
+      local metrics=(
+        "hyperhive.archive.prune.last_run_success=$ok"
+        "hyperhive.archive.prune.last_run_timestamp_seconds=$(date +%s)"
+      )
+      if snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json 2>/dev/null)" \
+        && count="$(jq 'length' <<<"$snaps" 2>/dev/null)"; then
+        metrics+=("hyperhive.archive.repo.snapshots=$count")
+        while read -r time; do
+          [ -n "$time" ] || continue
+          at="$(date -d "$time" +%s 2>/dev/null)" || continue
+          if [ -z "$latest" ] || [ "$at" -gt "$latest" ]; then latest="$at"; fi
+        done < <(jq -r '.[].time' <<<"$snaps")
+        if [ -n "$latest" ]; then
+          metrics+=("hyperhive.archive.repo.latest_snapshot_timestamp_seconds=$latest")
+        fi
+      fi
+      if stats="$(${restic}/bin/restic -r "$repo" --retry-lock 10m stats --mode raw-data --json 2>/dev/null)" \
+        && stats="$(jq -r '.total_size' <<<"$stats" 2>/dev/null)"; then
+        metrics+=("hyperhive.archive.repo.size_bytes=$stats")
+      fi
+      otlp_gauges swarm-archive-prune "agent=$agent" "''${metrics[@]}"
+    }
+
+
     store_init
     store_login
     if [ "$STORE_UP" != 1 ]; then
@@ -265,13 +298,16 @@ in
       fi
       repo=${repoOf}
       export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
+      ok=0
       if forget_future "$repo" "$agent" \
         && ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
         echo "pruned $agent"
+        ok=1
       else
         echo "archive prune failed: $agent" >&2
         failed=1
       fi
+      push_repo_metrics "$repo" "$agent" "$ok"
     done
     exit "$failed"
   '';

nix/host-modules/lib/otlp-gauges.nix

@@ -0,0 +1,49 @@
+# Shell text defining `otlp_gauges`, which pushes gauges to this host's OTLP/HTTP
+# metrics receiver:
+#
+#   otlp_gauges <service.name> <attr=value,attr=value> <metric=value>...
+#
+# One resource, one set of data-point attributes, one data point per metric.
+# Values are numbers. A failed build of the payload or a failed push prints a
+# warning on stderr and returns 0, so a unit's own result never depends on
+# telemetry. The swarm collector stamps `hive` on every sample it receives from
+# a hive, so callers do not set it.
+#
+# `endpoint` is the receiver's base URL, or null on a host without one, where
+# the function does nothing.
+#
+# Pure function, NOT a NixOS module. Needs `jq`, `date` and `curl` on the
+# unit's `path`.
+{ lib }:
+{ endpoint }:
+if endpoint == null then
+  ''
+    otlp_gauges() { :; }
+  ''
+else
+  ''
+    otlp_gauges() {
+      local service="$1" attrs="$2" body
+      shift 2
+      body="$(jq -n -c --arg service "$service" --arg attrs "$attrs" --arg now "$(date +%s%N)" '
+        ($attrs | split(",") | map(select(length > 0) | index("=") as $i
+          | {key: .[:$i], value: {stringValue: .[$i + 1:]}})) as $attributes
+        | {resourceMetrics: [{
+            resource: {attributes: [{key: "service.name", value: {stringValue: $service}}]},
+            scopeMetrics: [{metrics: ($ARGS.positional | map(index("=") as $i
+              | {name: .[:$i], gauge: {dataPoints: [{
+                  asDouble: (.[$i + 1:] | tonumber),
+                  attributes: $attributes,
+                  timeUnixNano: $now}]}}))}]}]}
+        ' --args "$@")" || {
+        echo "warning: could not build the metrics payload for $service: $*" >&2
+        return 0
+      }
+      printf '%s' "$body" |
+        curl --fail --silent --show-error --max-time 10 \
+          -H 'Content-Type: application/json' --data-binary @- \
+          ${lib.escapeShellArg "${endpoint}/v1/metrics"} ||
+        echo "warning: could not push metrics for $service" >&2
+      return 0
+    }
+  ''

nix/host-modules/swarm-archive.nix

@@ -61,6 +61,13 @@ let
       restic = cfg.resticPackage;
       addr = "127.0.0.1:${toString cfg.port}";
       authGroup = user;
+      # A host without a hive has no receiver on its own bridge; the swarm services
+      # there reach the swarm collector through a forwarder of their own.
+      otlpEndpoint =
+        if hyperhiveCfg.otel.collector.enable then
+          "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}"
+        else
+          null;
     }
     // (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; })
   );
@@ -408,6 +415,7 @@ in
         baoDeploy.package
         cfg.resticPackage
         pkgs.coreutils
+        pkgs.curl
         pkgs.gnugrep
         pkgs.jq
       ]

nix/host-modules/swarm-grafana.nix

@@ -269,6 +269,7 @@ let
       })
       [
         "agents.json"
+        "archive.json"
         "attic.json"
         "authelia.json"
         "bao.json"

nix/script-tests/otlp-gauges.nix

@@ -0,0 +1,27 @@
+# `checks.script-test-otlp-gauges` — runs ../host-modules/lib/otlp-gauges.nix's
+# `otlp_gauges` against a stub `curl`, with an endpoint and without. The cases
+# are in ./otlp-gauges.sh.
+{
+  pkgs,
+  ...
+}:
+let
+  render =
+    endpoint:
+    pkgs.writeText "otlp-gauges.sh" (
+      import ../host-modules/lib/otlp-gauges.nix { inherit (pkgs) lib; } { inherit endpoint; }
+    );
+in
+pkgs.runCommand "hyperhive-script-test-otlp-gauges"
+  {
+    nativeBuildInputs = [
+      pkgs.coreutils
+      pkgs.jq
+    ];
+    FUNCTIONS = render "http://10.42.0.1:4318";
+    FUNCTIONS_NO_ENDPOINT = render null;
+  }
+  ''
+    ${pkgs.bash}/bin/bash ${./otlp-gauges.sh}
+    touch "$out"
+  ''

nix/script-tests/otlp-gauges.sh

@@ -0,0 +1,73 @@
+# Cases for ../host-modules/lib/otlp-gauges.nix. `curl` is a stub that records
+# the payload and the URL it is given.
+set -uo pipefail
+
+failures=0
+
+fail() {
+  echo "FAILED: $case: $*" >&2
+  failures=$((failures + 1))
+}
+
+root=$(mktemp -d)
+mkdir -p "$root/bin"
+
+cat >"$root/bin/curl" <<STUB
+#!$BASH
+printf '%s\n' "\$*" >"$root/curl.args"
+cat >"$root/curl.payload"
+exit "\${STUB_CURL_STATUS:-0}"
+STUB
+chmod +x "$root/bin/curl"
+
+# Runs \`otlp_gauges "\$@"\` from file \$1 and returns its status.
+run() {
+  local functions="$1"
+  shift
+  rm -f "$root/curl.args" "$root/curl.payload"
+  PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; otlp_gauges "$@"' _ "$functions" "$@" \
+    >"$root/out" 2>"$root/err"
+}
+
+case="two gauges share the resource and the attributes"
+if run "$FUNCTIONS" hive-archive-backup 'agent=updo,tier=a=b' \
+  hyperhive.archive.backup.last_run_success=1 hyperhive.archive.backup.duration_seconds=12.5; then
+  p="$root/curl.payload"
+  [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$p")" = hive-archive-backup ] ||
+    fail "service.name: $(cat "$p")"
+  [ "$(jq -r '[.resourceMetrics[0].scopeMetrics[0].metrics[].name] | join(" ")' "$p")" = \
+    "hyperhive.archive.backup.last_run_success hyperhive.archive.backup.duration_seconds" ] ||
+    fail "names: $(cat "$p")"
+  [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[1].gauge.dataPoints[0].asDouble' "$p")" = 12.5 ] ||
+    fail "value: $(cat "$p")"
+  [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes | map("\(.key)=\(.value.stringValue)") | join(",")' "$p")" = \
+    "agent=updo,tier=a=b" ] || fail "attributes: $(cat "$p")"
+  grep -q 'http://10.42.0.1:4318/v1/metrics' "$root/curl.args" || fail "url: $(cat "$root/curl.args")"
+else
+  fail "the function failed: $(cat "$root/err")"
+fi
+
+case="a failing push warns and still returns 0"
+if STUB_CURL_STATUS=22 run "$FUNCTIONS" svc 'agent=x' m.a=1; then
+  grep -q 'warning: could not push' "$root/err" || fail "no warning: $(cat "$root/err")"
+else
+  fail "returned non-zero"
+fi
+
+case="a non-numeric value warns, pushes nothing and returns 0"
+if run "$FUNCTIONS" svc 'agent=x' m.a=oops; then
+  [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")"
+  grep -q 'warning: could not build' "$root/err" || fail "no warning: $(cat "$root/err")"
+else
+  fail "returned non-zero"
+fi
+
+case="without an endpoint nothing is pushed"
+if run "$FUNCTIONS_NO_ENDPOINT" svc 'agent=x' m.a=1; then
+  [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")"
+else
+  fail "returned non-zero"
+fi
+
+[ "$failures" -eq 0 ] || exit 1
+echo "all cases passed"

nix/vm-tests/swarm-archive.nix

@@ -109,6 +109,10 @@ let
           type = lib.types.listOf lib.types.str;
           default = [ ];
         };
+        otel.collector.enable = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+        };
         gateway = {
           enable = lib.mkOption {
             type = lib.types.bool;