diff --git a/docs/2-handbook/swarm/archive.md b/docs/2-handbook/swarm/archive.md index 2da8a37c6..f0bfe0562 100644 --- a/docs/2-handbook/swarm/archive.md +++ b/docs/2-handbook/swarm/archive.md @@ -134,6 +134,8 @@ On a hive, `systemctl status hive-archive-backup` shows the last run, and `journalctl -u hive-archive-backup` lists every agent it backed up or failed to back up. `systemctl list-timers hive-archive-backup` shows the next run. +The Grafana board `hyperhive · backups (restic)` shows the same per agent: the age of the last success, the last result, bytes added and snapshots kept. See [telemetry](../../4-implementation/swarm/telemetry.md). + To see an agent's snapshots, ask the archive with that agent's password, which the secret store holds at `swarm/hives//archive/` (fields `rest-password` and `restic-password`): diff --git a/docs/4-implementation/swarm/telemetry.md b/docs/4-implementation/swarm/telemetry.md index 2024624be..97fa08250 100644 --- a/docs/4-implementation/swarm/telemetry.md +++ b/docs/4-implementation/swarm/telemetry.md @@ -55,6 +55,7 @@ kept. - `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first. The **turns** tab holds what each agent's own `/stats` page shows about turns: average turn duration, cache hit rate and tokens per turn, turns per hour by agent and by model, the turn duration over time (median, 95th percentile and mean), and the mix of wake sources and results. It also has the tools called, the commands bash tasks ran, reminders per hour, the first-turn context size and fresh sessions per agent: everything the `/stats` page shows, except the per-bucket token bars the **cost** tab covers. The duration percentiles are right from the first deploy that includes the histogram bounds in `hive-agent`; before it, they read a flat 10 seconds. - `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent. +- `archive`: the restic backups of every agent to the swarm archive. Per agent: time since the last successful backup (red over two hours, twice the default hourly schedule), the last run's result, duration and bytes added, then snapshots, repository size and the last prune. Failures and recent runs come from the `hive-archive-backup` and `swarm-archive-prune` logs. The numbers are the `hyperhive.archive.backup.*` gauges, which each hive's backup unit pushes after every agent, and the `hyperhive.archive.prune.*` and `hyperhive.archive.repo.*` gauges, which the prune unit pushes after every repository when its host runs a hive. A hive that predates them shows no data for its agents; the log panels work regardless. Filter by hive and agent. - `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue. - `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents. - `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, the busiest agents and destinations, and data transferred by domain over time (the top 10 domains plus an other line, summed from the proxy's access log in VictoriaLogs; the proxy logs a CONNECT tunnel when it closes, so a long-lived connection shows as one spike at close). **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent and by domain, to see which agents queried a domain and when. diff --git a/nix/checks.nix b/nix/checks.nix index e569439e7..046ad8e3e 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -80,6 +80,9 @@ in # Runs ./host-modules/lib/database-size-push.nix against a symlinked directory, # as a dynamic user's state directory is: the reported size is the target's. script-test-database-size-push = import ./script-tests/database-size-push.nix { inherit pkgs; }; + # Runs ./host-modules/lib/otlp-gauges.nix's `otlp_gauges` against a stub `curl`: the + # payload shape, a failed push and a bad value both return 0, no endpoint pushes nothing. + script-test-otlp-gauges = import ./script-tests/otlp-gauges.nix { inherit pkgs; }; # Runs ./host-modules/lib/retry-in-job.nix's `retry_in_job`, which the agent # trust-bundle fetch retries through inside its start job: success after two # failures, the wait schedule, and a return of 1 once the budget is spent. diff --git a/nix/host-modules/hive-archive-backup.nix b/nix/host-modules/hive-archive-backup.nix index ae47985ac..191508b05 100644 --- a/nix/host-modules/hive-archive-backup.nix +++ b/nix/host-modules/hive-archive-backup.nix @@ -19,6 +19,9 @@ # A failed agent prints `archive backup failed: ` to the journal, # which the swarm's `archive backup failed` alert reads, and fails the unit # after the others have been tried. +# +# Each agent's run also pushes `hyperhive.archive.backup.*` gauges to this +# hive's collector (./lib/archive-backup.nix); a failed push is only a warning. { pkgs, lib, @@ -56,6 +59,7 @@ let agentRoot = "/var/lib/hyperhive/agents"; url = "https://${hyperhiveCfg.swarm.archive.ownDomain}"; inherit caBundle; + otlpEndpoint = "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}"; }; in { @@ -137,6 +141,7 @@ in baoDeploy.package pkgs.btrfs-progs pkgs.coreutils + pkgs.curl pkgs.gnugrep pkgs.jq ]; diff --git a/nix/host-modules/lib/archive-backup.nix b/nix/host-modules/lib/archive-backup.nix index 8af4f0f29..f5abf1c0a 100644 --- a/nix/host-modules/lib/archive-backup.nix +++ b/nix/host-modules/lib/archive-backup.nix @@ -4,11 +4,15 @@ # wraps it in a unit, and the VM test (nix/vm-tests/swarm-archive.nix) runs the # same script against a stand-in `bao`. # -# Needs `bao`, `jq`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path` +# Needs `bao`, `jq`, `curl`, `coreutils`, `gnugrep`, `btrfs-progs` on the unit's `path` # and the `BAO_*` client set. A store that cannot be reached, or an agent whose # backup fails, prints a line starting # `archive backup failed`, which the swarm's alert reads, and ends the run with # a non-zero status once the other agents have been tried. +# +# After each agent it pushes `hyperhive.archive.backup.*` gauges (`agent` attribute) +# through ./otlp-gauges.nix: last run and last success time, result, wall time, +# bytes added and files new and changed. A failed agent still reports its result. { lib }: { restic, @@ -25,11 +29,15 @@ excludes, # A CA file to trust instead of restic's default, or null. caBundle, + # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null. + otlpEndpoint ? null, }: let storeSeededSecret = import ./store-seeded-secret.nix { }; grants = hive: "secret/swarm/hives/${hive}/archive"; + + otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; }; in '' set -euo pipefail @@ -45,6 +53,7 @@ in ${lib.optionalString (caBundle != null) '' export RESTIC_CACERT=${lib.escapeShellArg caBundle} ''} + ${otlpGauges} root=${lib.escapeShellArg agentRoot} grants=${lib.escapeShellArg (grants hive)} @@ -83,9 +92,14 @@ in if [ "''${#targets[@]}" = 0 ]; then echo "$agent has none of its backup paths yet; nothing to back up." else - ${restic}/bin/restic backup --host "$agent" --tag archive --exclude-caches \ + # `--json` for the summary line the metrics read. Only that line and the + # errors are printed; the progress lines are dropped. + ${restic}/bin/restic backup --json --host "$agent" --tag archive --exclude-caches \ ${lib.concatMapStringsSep " " (e: "--exclude ${lib.escapeShellArg e}") excludes} \ - -- "''${targets[@]}" || rc=$? + -- "''${targets[@]}" | + jq -R -r '. as $line | ((fromjson? | select(type == "object")) // null) as $j + | if $j == null or $j.message_type == "error" or $j.message_type == "summary" + then $line else empty end' || rc=$? fi if [ -n "$snap" ]; then btrfs subvolume delete "$snap" >/dev/null || rc=1 @@ -93,6 +107,29 @@ in return "$rc" } + # One agent's run as gauges: `$5` is its output, whose `summary` line has the + # counts when restic got that far. Never fails the run. + push_backup_metrics() { + local agent="$1" ok="$2" started="$3" finished="$4" out="$5" summary + local metrics=( + "hyperhive.archive.backup.last_run_timestamp_seconds=$finished" + "hyperhive.archive.backup.last_run_success=$ok" + "hyperhive.archive.backup.duration_seconds=$((finished - started))" + ) + if [ "$ok" = 1 ]; then + metrics+=("hyperhive.archive.backup.last_success_timestamp_seconds=$finished") + summary="$(printf '%s\n' "$out" | jq -R -r ' + fromjson? | select(type == "object" and .message_type == "summary") + | "hyperhive.archive.backup.data_added_bytes=\(.data_added)", + "hyperhive.archive.backup.files_new=\(.files_new)", + "hyperhive.archive.backup.files_changed=\(.files_changed)"' 2>/dev/null)" || summary="" + while IFS= read -r line; do + [ -z "$line" ] || metrics+=("$line") + done <<<"$summary" + fi + otlp_gauges hive-archive-backup "agent=$agent" "''${metrics[@]}" + } + # The agents the controller granted this hive; none yet is "No value found". rc=0 granted="$(store_bao kv list -format=json "$grants" 2>"$STORE_ERR")" || rc=$? @@ -119,13 +156,19 @@ in echo "$agent has no archive credentials on this hive; skipping." continue fi - if (backup_agent "$agent"); then + started="$(date +%s)" + ok=0 + out="$(backup_agent "$agent")" && ok=1 + [ -z "$out" ] || printf '%s\n' "$out" + finished="$(date +%s)" + if [ "$ok" = 1 ]; then backed_up=$((backed_up + 1)) echo "backed up $agent" else echo "archive backup failed: $agent" >&2 failed=1 fi + push_backup_metrics "$agent" "$ok" "$started" "$finished" "$out" done # No grant for any agent here: the swarm has no archive, or its controller diff --git a/nix/host-modules/lib/archive-server.nix b/nix/host-modules/lib/archive-server.nix index d05dd53e8..13da635ec 100644 --- a/nix/host-modules/lib/archive-server.nix +++ b/nix/host-modules/lib/archive-server.nix @@ -31,11 +31,14 @@ authGroup, # `{ keepWithin, keepLast }`; a null `keepWithin` leaves that flag off. retention, + # The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null. + otlpEndpoint ? null, }: let atomicWriteSecret = import ./atomic-write-secret.nix { }; refreshConsumer = import ./refresh-consumer.nix { }; storeSeededSecret = import ./store-seeded-secret.nix { }; + otlpGauges = import ./otlp-gauges.nix { inherit lib; } { endpoint = otlpEndpoint; }; usersPrefix = "secret/swarm/services/archive/users"; resticPrefix = "secret/swarm/services/archive/restic"; @@ -242,6 +245,36 @@ in fi } + ${otlpGauges} + # Pushes one repository's gauges: the outcome of this prune, the snapshot + # count and newest snapshot time, and the size of the data in the repository. + # Reads the repository after the prune. Never fails the run. + push_repo_metrics() { + local repo="$1" agent="$2" ok="$3" snaps stats time at latest="" count + local metrics=( + "hyperhive.archive.prune.last_run_success=$ok" + "hyperhive.archive.prune.last_run_timestamp_seconds=$(date +%s)" + ) + if snaps="$(${restic}/bin/restic -r "$repo" --retry-lock 10m snapshots --json 2>/dev/null)" \ + && count="$(jq 'length' <<<"$snaps" 2>/dev/null)"; then + metrics+=("hyperhive.archive.repo.snapshots=$count") + while read -r time; do + [ -n "$time" ] || continue + at="$(date -d "$time" +%s 2>/dev/null)" || continue + if [ -z "$latest" ] || [ "$at" -gt "$latest" ]; then latest="$at"; fi + done < <(jq -r '.[].time' <<<"$snaps") + if [ -n "$latest" ]; then + metrics+=("hyperhive.archive.repo.latest_snapshot_timestamp_seconds=$latest") + fi + fi + if stats="$(${restic}/bin/restic -r "$repo" --retry-lock 10m stats --mode raw-data --json 2>/dev/null)" \ + && stats="$(jq -r '.total_size' <<<"$stats" 2>/dev/null)"; then + metrics+=("hyperhive.archive.repo.size_bytes=$stats") + fi + otlp_gauges swarm-archive-prune "agent=$agent" "''${metrics[@]}" + } + + store_init store_login if [ "$STORE_UP" != 1 ]; then @@ -265,13 +298,16 @@ in fi repo=${repoOf} export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent" + ok=0 if forget_future "$repo" "$agent" \ && ${restic}/bin/restic -r "$repo" --retry-lock 10m forget --group-by host ${keepFlags} --prune; then echo "pruned $agent" + ok=1 else echo "archive prune failed: $agent" >&2 failed=1 fi + push_repo_metrics "$repo" "$agent" "$ok" done exit "$failed" ''; diff --git a/nix/host-modules/lib/otlp-gauges.nix b/nix/host-modules/lib/otlp-gauges.nix new file mode 100644 index 000000000..e9ec410bb --- /dev/null +++ b/nix/host-modules/lib/otlp-gauges.nix @@ -0,0 +1,49 @@ +# Shell text defining `otlp_gauges`, which pushes gauges to this host's OTLP/HTTP +# metrics receiver: +# +# otlp_gauges ... +# +# One resource, one set of data-point attributes, one data point per metric. +# Values are numbers. A failed build of the payload or a failed push prints a +# warning on stderr and returns 0, so a unit's own result never depends on +# telemetry. The swarm collector stamps `hive` on every sample it receives from +# a hive, so callers do not set it. +# +# `endpoint` is the receiver's base URL, or null on a host without one, where +# the function does nothing. +# +# Pure function, NOT a NixOS module. Needs `jq`, `date` and `curl` on the +# unit's `path`. +{ lib }: +{ endpoint }: +if endpoint == null then + '' + otlp_gauges() { :; } + '' +else + '' + otlp_gauges() { + local service="$1" attrs="$2" body + shift 2 + body="$(jq -n -c --arg service "$service" --arg attrs "$attrs" --arg now "$(date +%s%N)" ' + ($attrs | split(",") | map(select(length > 0) | index("=") as $i + | {key: .[:$i], value: {stringValue: .[$i + 1:]}})) as $attributes + | {resourceMetrics: [{ + resource: {attributes: [{key: "service.name", value: {stringValue: $service}}]}, + scopeMetrics: [{metrics: ($ARGS.positional | map(index("=") as $i + | {name: .[:$i], gauge: {dataPoints: [{ + asDouble: (.[$i + 1:] | tonumber), + attributes: $attributes, + timeUnixNano: $now}]}}))}]}]} + ' --args "$@")" || { + echo "warning: could not build the metrics payload for $service: $*" >&2 + return 0 + } + printf '%s' "$body" | + curl --fail --silent --show-error --max-time 10 \ + -H 'Content-Type: application/json' --data-binary @- \ + ${lib.escapeShellArg "${endpoint}/v1/metrics"} || + echo "warning: could not push metrics for $service" >&2 + return 0 + } + '' diff --git a/nix/host-modules/swarm-archive.nix b/nix/host-modules/swarm-archive.nix index 457350579..e7ad9c571 100644 --- a/nix/host-modules/swarm-archive.nix +++ b/nix/host-modules/swarm-archive.nix @@ -61,6 +61,13 @@ let restic = cfg.resticPackage; addr = "127.0.0.1:${toString cfg.port}"; authGroup = user; + # A host without a hive has no receiver on its own bridge; the swarm services + # there reach the swarm collector through a forwarder of their own. + otlpEndpoint = + if hyperhiveCfg.otel.collector.enable then + "http://${hyperhiveCfg.network.bridgeIp}:${toString hyperhiveCfg.swarm.hiveOtel.collectorPort}" + else + null; } // (if useS3 then { inherit (cfg) s3; } else { inherit reposDir; }) ); @@ -408,6 +415,7 @@ in baoDeploy.package cfg.resticPackage pkgs.coreutils + pkgs.curl pkgs.gnugrep pkgs.jq ] diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 6b9611667..9c712633d 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -269,6 +269,7 @@ let }) [ "agents.json" + "archive.json" "attic.json" "authelia.json" "bao.json" diff --git a/nix/host-modules/swarm-grafana/dashboards/archive.json b/nix/host-modules/swarm-grafana/dashboards/archive.json new file mode 100644 index 000000000..9b0edb001 --- /dev/null +++ b/nix/host-modules/swarm-grafana/dashboards/archive.json @@ -0,0 +1,1032 @@ +{ + "title": "hyperhive · backups (restic)", + "uid": "hyperhive-archive", + "description": "Each agent's restic backup to the swarm archive, and the archive's prune.", + "editable": true, + "refresh": "1m", + "schemaVersion": 39, + "tags": ["hyperhive", "swarm", "archive", "backup", "restic"], + "time": { + "from": "now-24h", + "to": "now" + }, + "timezone": "utc", + "version": 1, + "templating": { + "list": [ + { + "name": "hive", + "label": "Hive", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\"}, hive)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "agent", + "label": "Agent", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\"}, agent)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Backups", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Agents past 2× the interval", + "description": "Agents whose last successful backup is over 2 hours old, twice the hourly schedule.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "count(time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) > 7200) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 3, + "type": "stat", + "title": "Agents whose last run failed", + "description": "Agents whose latest backup run ended in failure. Source: last_run_success gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) == 0) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "red" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 4, + "type": "stat", + "title": "Agents backed up", + "description": "Agents that reported a backup run in the last 7 days. Source: last_run_success gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "green" + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "graphMode": "none", + "colorMode": "value", + "textMode": "value", + "justifyMode": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + } + } + }, + { + "id": 5, + "type": "table", + "title": "Last backup per agent", + "description": "Per agent: time since last success (red over 2 hours), last result, duration, data added.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "agent", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true + }, + "renameByName": { + "agent": "Agent", + "Value #A": "Since last success", + "Value #B": "Last result", + "Value #C": "Duration", + "Value #D": "Data added" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Since last success" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 7200 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Last result" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "failed", + "color": "red", + "index": 0 + }, + "1": { + "text": "ok", + "color": "green", + "index": 1 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Data added" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Since last success", + "desc": true + } + ] + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Data added per run", + "description": "Bytes each backup run added to the repository, per agent. Source: data_added_bytes gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 14, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Backup duration", + "description": "Wall time of each agent's backup run, snapshot included. Source: duration_seconds gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 12, + "y": 14, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "row", + "title": "Repositories", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 9, + "type": "table", + "title": "Repository and prune per agent", + "description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.latest_snapshot_timestamp_seconds\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_timestamp_seconds\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + }, + { + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "agent", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true + }, + "renameByName": { + "agent": "Agent", + "Value #A": "Snapshots", + "Value #B": "Size", + "Value #C": "Newest snapshot age", + "Value #D": "Since last prune", + "Value #E": "Prune result" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Newest snapshot age" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since last prune" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Prune result" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "failed", + "color": "red", + "index": 0 + }, + "1": { + "text": "ok", + "color": "green", + "index": 1 + } + } + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Agent", + "desc": true + } + ] + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Snapshots per repository", + "description": "Snapshots each agent's repository holds after the prune. Source: repo.snapshots gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Repository size", + "description": "Size of the data in each agent's repository. Source: repo.size_bytes gauge.", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "@datasourceUid@" + }, + "expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"})", + "legendFormat": "{{agent}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "row", + "title": "Failures", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 13, + "type": "timeseries", + "title": "Failures over time", + "description": "Backup and prune failure lines per interval.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\" | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "backup failed" + }, + { + "refId": "B", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\" | stats count_uniq(__CURSOR) as n", + "queryType": "statsRange", + "legendFormat": "prune failed" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 60, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "logs", + "title": "Recent backup failures", + "description": "Agents hive-archive-backup could not back up, newest first.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "x": 12, + "y": 41, + "w": 12, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\"", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + }, + { + "id": 15, + "type": "logs", + "title": "Recent prune failures", + "description": "Repositories swarm-archive-prune could not prune, newest first.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 12, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\"", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + }, + { + "id": 16, + "type": "logs", + "title": "Recent backup runs", + "description": "Agents backed up, skipped and failed by hive-archive-backup, newest first.", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "@logsDatasourceUid@" + }, + "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\"", + "queryType": "instant" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + } + } + ] +} diff --git a/nix/script-tests/otlp-gauges.nix b/nix/script-tests/otlp-gauges.nix new file mode 100644 index 000000000..eec37f7c9 --- /dev/null +++ b/nix/script-tests/otlp-gauges.nix @@ -0,0 +1,27 @@ +# `checks.script-test-otlp-gauges` — runs ../host-modules/lib/otlp-gauges.nix's +# `otlp_gauges` against a stub `curl`, with an endpoint and without. The cases +# are in ./otlp-gauges.sh. +{ + pkgs, + ... +}: +let + render = + endpoint: + pkgs.writeText "otlp-gauges.sh" ( + import ../host-modules/lib/otlp-gauges.nix { inherit (pkgs) lib; } { inherit endpoint; } + ); +in +pkgs.runCommand "hyperhive-script-test-otlp-gauges" + { + nativeBuildInputs = [ + pkgs.coreutils + pkgs.jq + ]; + FUNCTIONS = render "http://10.42.0.1:4318"; + FUNCTIONS_NO_ENDPOINT = render null; + } + '' + ${pkgs.bash}/bin/bash ${./otlp-gauges.sh} + touch "$out" + '' diff --git a/nix/script-tests/otlp-gauges.sh b/nix/script-tests/otlp-gauges.sh new file mode 100644 index 000000000..40d96b951 --- /dev/null +++ b/nix/script-tests/otlp-gauges.sh @@ -0,0 +1,73 @@ +# Cases for ../host-modules/lib/otlp-gauges.nix. `curl` is a stub that records +# the payload and the URL it is given. +set -uo pipefail + +failures=0 + +fail() { + echo "FAILED: $case: $*" >&2 + failures=$((failures + 1)) +} + +root=$(mktemp -d) +mkdir -p "$root/bin" + +cat >"$root/bin/curl" <"$root/curl.args" +cat >"$root/curl.payload" +exit "\${STUB_CURL_STATUS:-0}" +STUB +chmod +x "$root/bin/curl" + +# Runs \`otlp_gauges "\$@"\` from file \$1 and returns its status. +run() { + local functions="$1" + shift + rm -f "$root/curl.args" "$root/curl.payload" + PATH="$root/bin:$PATH" bash -c 'source "$1"; shift; otlp_gauges "$@"' _ "$functions" "$@" \ + >"$root/out" 2>"$root/err" +} + +case="two gauges share the resource and the attributes" +if run "$FUNCTIONS" hive-archive-backup 'agent=updo,tier=a=b' \ + hyperhive.archive.backup.last_run_success=1 hyperhive.archive.backup.duration_seconds=12.5; then + p="$root/curl.payload" + [ "$(jq -r '.resourceMetrics[0].resource.attributes[0].value.stringValue' "$p")" = hive-archive-backup ] || + fail "service.name: $(cat "$p")" + [ "$(jq -r '[.resourceMetrics[0].scopeMetrics[0].metrics[].name] | join(" ")' "$p")" = \ + "hyperhive.archive.backup.last_run_success hyperhive.archive.backup.duration_seconds" ] || + fail "names: $(cat "$p")" + [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[1].gauge.dataPoints[0].asDouble' "$p")" = 12.5 ] || + fail "value: $(cat "$p")" + [ "$(jq -r '.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes | map("\(.key)=\(.value.stringValue)") | join(",")' "$p")" = \ + "agent=updo,tier=a=b" ] || fail "attributes: $(cat "$p")" + grep -q 'http://10.42.0.1:4318/v1/metrics' "$root/curl.args" || fail "url: $(cat "$root/curl.args")" +else + fail "the function failed: $(cat "$root/err")" +fi + +case="a failing push warns and still returns 0" +if STUB_CURL_STATUS=22 run "$FUNCTIONS" svc 'agent=x' m.a=1; then + grep -q 'warning: could not push' "$root/err" || fail "no warning: $(cat "$root/err")" +else + fail "returned non-zero" +fi + +case="a non-numeric value warns, pushes nothing and returns 0" +if run "$FUNCTIONS" svc 'agent=x' m.a=oops; then + [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")" + grep -q 'warning: could not build' "$root/err" || fail "no warning: $(cat "$root/err")" +else + fail "returned non-zero" +fi + +case="without an endpoint nothing is pushed" +if run "$FUNCTIONS_NO_ENDPOINT" svc 'agent=x' m.a=1; then + [ ! -e "$root/curl.payload" ] || fail "pushed $(cat "$root/curl.payload")" +else + fail "returned non-zero" +fi + +[ "$failures" -eq 0 ] || exit 1 +echo "all cases passed" diff --git a/nix/vm-tests/swarm-archive.nix b/nix/vm-tests/swarm-archive.nix index 8f7320ec0..7c6ab5f0d 100644 --- a/nix/vm-tests/swarm-archive.nix +++ b/nix/vm-tests/swarm-archive.nix @@ -109,6 +109,10 @@ let type = lib.types.listOf lib.types.str; default = [ ]; }; + otel.collector.enable = lib.mkOption { + type = lib.types.bool; + default = false; + }; gateway = { enable = lib.mkOption { type = lib.types.bool;