archive: push restic backup and prune metrics, and add a backups dashboard

nix/host-modules/hive-archive-backup.nix

@@ -19,6 +19,9 @@
 # A failed agent prints `archive backup failed: <agent>` to the journal,
 # which the swarm's `archive backup failed` alert reads, and fails the unit
 # after the others have been tried.
+#
+# Each agent's run also pushes `hyperhive.archive.backup.*` gauges to this
+# hive's collector (./lib/archive-backup.nix); a failed push is only a warning.
 {
   pkgs,
   lib,
@@ -56,6 +59,7 @@ let
     agentRoot = "/var/lib/hyperhive/agents";
     url = "https://${hyperhiveCfg.swarm.archive.ownDomain}";
     inherit caBundle;
+    otlpEndpoint = "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}";
   };
 in
 {
@@ -137,6 +141,7 @@ in
         baoDeploy.package
         pkgs.btrfs-progs
         pkgs.coreutils
+        pkgs.curl
         pkgs.gnugrep
         pkgs.jq
       ];

nix/host-modules/lib/archive-backup.nix

@@ -4,11 +4,15 @@
 # wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the
 # same script against a stand-in `bao`.
 #
-# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
+# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path`
 # and the `BAO_*` client set. A store that cannot be reached, or an agent whose
 # backup fails, prints a line starting
 # `archive backup failed`, which the swarm's alert reads, and ends the run with
 # a non-zero status once the other agents have been tried.
+#
+# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute)
+# through ./otlp-gauges.nix: last run and last success time, result, wall time,
+# bytes added and files new and changed. A failed agent still reports its result.
 { lib }:
 {
   restic,
@@ -25,11 +29,15 @@
   excludes,
   # A CA file to trust instead of restic's default, or null.
   caBundle,
+  # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
+  otlpEndpoint ? null,
 }:
 let
   storeSeededSecret = import ./store-seeded-secret.nix { };
 
   grants = hive: "secret/swarm/hives/${hive}/archive";
+
+  otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
 in
 ''
   set -euo pipefail
@@ -45,6 +53,7 @@ in
   ${lib.optionalString (caBundle != null) ''
     export RESTIC_CACERT=${lib.escapeShellArg caBundle}
   ''}
+  ${otlpGauges}
   root=${lib.escapeShellArg agentRoot}
   grants=${lib.escapeShellArg (grants hive)}
 
@@ -83,9 +92,14 @@ in
     if [ "''${#targets[@]}" = 0 ]; then
       echo "$agent has none of its backup paths yet; nothing to back up."
     else
-      ${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \
+      # `--json` for the summary line the metrics read. Only that line and the
+      # errors are printed; the progress lines are dropped.
+      ${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \
         ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \
-        -- "''${targets[@]}" || rc=$?
+        -- "''${targets[@]}" |
+        jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j
+          | if $j == null or $j.message_type == "error" or $j.message_type == "summary"
+            then $line else empty end' || rc=$?
     fi
     if [ -n "$snap" ]; then
       btrfs subvolume delete "$snap" >/dev/null || rc=1
@@ -93,6 +107,29 @@ in
     return "$rc"
   }
 
+  # One agent's run as gauges: `$5` is its output, whose `summary` line has the
+  # counts when restic got that far. Never fails the run.
+  push_backup_metrics() {
+    local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary
+    local metrics=(
+      "hyperhive.archive.backup.last_run_timestamp_seconds=$finished"
+      "hyperhive.archive.backup.last_run_success=$ok"
+      "hyperhive.archive.backup.duration_seconds=$((finished - started))"
+    )
+    if [ "$ok" = 1 ]; then
+      metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished")
+      summary="$(printf '%s\n' "$out" | jq -R -r '
+        fromjson? | select(type == "object" and .message_type == "summary")
+        | "hyperhive.archive.backup.data_added_bytes=\(.data_added)",
+          "hyperhive.archive.backup.files_new=\(.files_new)",
+          "hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary=""
+      while IFS= read -r line; do
+        [ -z "$line" ] || metrics+=("$line")
+      done <<<"$summary"
+    fi
+    otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}"
+  }
+
   # The agents the controller granted this hive; none yet is "No value found".
   rc=0
   granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$?
@@ -119,13 +156,19 @@ in
       echo "$agent has no archive credentials on this hive; skipping."
       continue
     fi
-    if (backup_agent "$agent"); then
+    started="$(date +%s)"
+    ok=0
+    out="$(backup_agent "$agent")" && ok=1
+    [ -z "$out" ] || printf '%s\n' "$out"
+    finished="$(date +%s)"
+    if [ "$ok" = 1 ]; then
       backed_up=$((backed_up + 1))
       echo "backed up $agent"
     else
       echo "archive backup failed: $agent" >&2
       failed=1
     fi
+    push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out"
   done
 
   # No grant for any agent here: the swarm has no archive, or its controller

nix/host-modules/lib/archive-server.nix

@@ -31,11 +31,14 @@
   authGroup,
   # `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off.
   retention,
+  # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
+  otlpEndpoint ? null,
 }:
 let
   atomicWriteSecret = import ./atomic-write-secret.nix { };
   refreshConsumer = import ./refresh-consumer.nix { };
   storeSeededSecret = import ./store-seeded-secret.nix { };
+  otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; };
 
   usersPrefix = "secret/swarm/services/archive/users";
   resticPrefix = "secret/swarm/services/archive/restic";
@@ -242,6 +245,36 @@ in
       fi
     }
 
+    ${otlpGauges}
+    # Pushes one repository's gauges: the outcome of this prune, the snapshot
+    # count and newest snapshot time, and the size of the data in the repository.
+    # Reads the repository after the prune. Never fails the run.
+    push_repo_metrics() {
+      local repo="$1" agent="$2" ok="$3" snaps stats time at latest="" count
+      local metrics=(
+        "hyperhive.archive.prune.last_run_success=$ok"
+        "hyperhive.archive.prune.last_run_timestamp_seconds=$(date +%s)"
+      )
+      if snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json 2>/dev/null)" \
+        && count="$(jq 'length' <<<"$snaps" 2>/dev/null)"; then
+        metrics+=("hyperhive.archive.repo.snapshots=$count")
+        while read -r time; do
+          [ -n "$time" ] || continue
+          at="$(date -d "$time" +%s 2>/dev/null)" || continue
+          if [ -z "$latest" ] || [ "$at" -gt "$latest" ]; then latest="$at"; fi
+        done < <(jq -r '.[].time' <<<"$snaps")
+        if [ -n "$latest" ]; then
+          metrics+=("hyperhive.archive.repo.latest_snapshot_timestamp_seconds=$latest")
+        fi
+      fi
+      if stats="$(${restic}/bin/restic -r "$repo" --retry-lock 10m stats --mode raw-data --json 2>/dev/null)" \
+        && stats="$(jq -r '.total_size' <<<"$stats" 2>/dev/null)"; then
+        metrics+=("hyperhive.archive.repo.size_bytes=$stats")
+      fi
+      otlp_gauges swarm-archive-prune "agent=$agent" "''${metrics[@]}"
+    }
+
+
     store_init
     store_login
     if [ "$STORE_UP" != 1 ]; then
@@ -265,13 +298,16 @@ in
       fi
       repo=${repoOf}
       export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
+      ok=0
       if forget_future "$repo" "$agent" \
         && ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then
         echo "pruned $agent"
+        ok=1
       else
         echo "archive prune failed: $agent" >&2
         failed=1
       fi
+      push_repo_metrics "$repo" "$agent" "$ok"
     done
     exit "$failed"
   '';

nix/host-modules/lib/otlp-gauges.nix

@@ -0,0 +1,49 @@
+# Shell text defining `otlp_gauges`, which pushes gauges to this host's OTLP/HTTP
+# metrics receiver:
+#
+#   otlp_gauges <service.name> <attr=value,attr=value> <metric=value>...
+#
+# One resource, one set of data-point attributes, one data point per metric.
+# Values are numbers. A failed build of the payload or a failed push prints a
+# warning on stderr and returns 0, so a unit's own result never depends on
+# telemetry. The swarm collector stamps `hive` on every sample it receives from
+# a hive, so callers do not set it.
+#
+# `endpoint` is the receiver's base URL, or null on a host without one, where
+# the function does nothing.
+#
+# Pure function, NOT a NixOS module. Needs `jq`, `date` and `curl` on the
+# unit's `path`.
+{ lib }:
+{ endpoint }:
+if endpoint == null then
+  ''
+    otlp_gauges() { :; }
+  ''
+else
+  ''
+    otlp_gauges() {
+      local service="$1" attrs="$2" body
+      shift 2
+      body="$(jq -n -c --arg service "$service" --arg attrs "$attrs" --arg now "$(date +%s%N)" '
+        ($attrs | split(",") | map(select(length > 0) | index("=") as $i
+          | {key: .[:$i], value: {stringValue: .[$i + 1:]}})) as $attributes
+        | {resourceMetrics: [{
+            resource: {attributes: [{key: "service.name", value: {stringValue: $service}}]},
+            scopeMetrics: [{metrics: ($ARGS.positional | map(index("=") as $i
+              | {name: .[:$i], gauge: {dataPoints: [{
+                  asDouble: (.[$i + 1:] | tonumber),
+                  attributes: $attributes,
+                  timeUnixNano: $now}]}}))}]}]}
+        ' --args "$@")" || {
+        echo "warning: could not build the metrics payload for $service: $*" >&2
+        return 0
+      }
+      printf '%s' "$body" |
+        curl --fail --silent --show-error --max-time 10 \
+          -H 'Content-Type: application/json' --data-binary @- \
+          ${lib.escapeShellArg "${endpoint}/v1/metrics"} ||
+        echo "warning: could not push metrics for $service" >&2
+      return 0
+    }
+  ''

nix/host-modules/swarm-archive.nix

@@ -61,6 +61,13 @@ let
       restic = cfg.resticPackage;
       addr = "127.0.0.1:${toString cfg.port}";
       authGroup = user;
+      # A host without a hive has no receiver on its own bridge; the swarm services
+      # there reach the swarm collector through a forwarder of their own.
+      otlpEndpoint =
+        if hyperhiveCfg.otel.collector.enable then
+          "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}"
+        else
+          null;
     }
     // (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; })
   );
@@ -408,6 +415,7 @@ in
         baoDeploy.package
         cfg.resticPackage
         pkgs.coreutils
+        pkgs.curl
         pkgs.gnugrep
         pkgs.jq
       ]

nix/host-modules/swarm-grafana.nix

@@ -269,6 +269,7 @@ let
       })
       [
         "agents.json"
+        "archive.json"
         "attic.json"
         "authelia.json"
         "bao.json"