mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-archive: check each agent's restic repository weekly

Browse source 
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas 2026-11-14 11:51:25 +01:00
commit 5c30985a25
6 changed files with 503 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -100,6 +100,10 @@ let
here = deployCfg.archive.enable;
policies = [ "swarm-bao-archive-policy" ];
};
swarm-archive-check = {
here = deployCfg.archive.enable;
policies = [ "swarm-bao-archive-policy" ];
};
# ./hive-matrix.nix, under matrix-ctl's leaf.
swarm-bao-matrix-oidc = {
here = deployCfg.matrix.enable;

1
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -72,6 +72,7 @@ let
# ./swarm-archive.nix
swarm-archive-users = deployCfg.archive.enable;
swarm-archive-prune = deployCfg.archive.enable;
swarm-archive-check = deployCfg.archive.enable;
# ./swarm-otel.nix
swarm-bao-otel-oidc =
deployCfg.swarm-otel.enable

82
nix/host-modules/lib/archive-server.nix
View file

@ -1,6 +1,6 @@
# The archive server's moving parts: the rclone command that serves the
# repositories, the script that renders its users file from the secret store,
# and the script that prunes each repository. A pure function, NOT a NixOS
# the script that prunes each repository and the one that checks them. A pure function, NOT a NixOS
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
# `bao`.
@ -33,6 +33,9 @@
retention,
# The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
otlpEndpoint ? null,
# `restic check --read-data-subset=<this>`: what share of each repository's data the
# check reads, e.g. `5%`.
checkSubset ? "5%",
}:
let
atomicWriteSecret = import ./atomic-write-secret.nix { };
@ -46,6 +49,16 @@ let
s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
# Takes the lock the prune and the check share, when the unit sets
# `ARCHIVE_MAINTENANCE_LOCK` (a path in a directory both units can write); waits
# for the holder. Held until the script exits. Needs `flock` on the unit's `path`.
maintenanceLock = ''
if [ -n "''${ARCHIVE_MAINTENANCE_LOCK:-}" ]; then
exec 9>"$ARCHIVE_MAINTENANCE_LOCK"
flock 9
fi
'';
# Slack for the clocks of the hives that stamp snapshots.
futureSkewSeconds = 3600;
@ -223,6 +236,7 @@ in
pruneScript = ''
set -euo pipefail
${storeSeededSecret}
${maintenanceLock}
# Forgets the snapshots of $repo dated after the limit, naming them in the
# log; returns non-zero when it cannot read the list or a time.
@ -311,4 +325,70 @@ in
done
exit "$failed"
'';
# `restic check` on every repository, reading `checkSubset` of its data: a random
# share each run, so repeated runs cover the repository without any run
# reading all of it. The repository's structure (index, snapshots, trees) is
# always checked in full. One repository's failure does not stop the others;
# the run exits non-zero if any failed.
checkScript = ''
set -euo pipefail
${storeSeededSecret}
${maintenanceLock}
${otlpGauges}
# Pushes the outcome of one repository's check. Never fails the run.
push_check_metrics() {
local agent="$1" ok="$2" start="$3" end="$4"
otlp_gauges swarm-archive-check "agent=$agent" \
"hyperhive.archive.check.last_run_success=$ok" \
"hyperhive.archive.check.last_run_timestamp_seconds=$end" \
"hyperhive.archive.check.duration_seconds=$(( end - start ))"
}
# Checks the repository of each agent named in the arguments; returns
# non-zero if any check failed.
check_agents() {
local agent repo start ok failed=0
for agent in "$@"; do
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "archive check failed: $agent is not an agent name" >&2
failed=1
continue
fi
repo=${repoOf}
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
ok=0
start="$(date +%s)"
if ${restic}/bin/restic -r "$repo" --retry-lock 10m check --read-data-subset=${lib.escapeShellArg checkSubset}; then
echo "checked $agent"
ok=1
else
echo "archive check failed: $agent" >&2
failed=1
fi
push_check_metrics "$agent" "$ok" "$start" "$(date +%s)"
done
return "$failed"
}
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "archive check failed: could not log in to the swarm secret store" >&2
exit 1
fi
${lib.optionalString (s3 != null) s3Credentials}
list_agents() {
${listAgents}
}
# Under `set -e` a failed listing ends the run; inside the `for` it would not.
agents="$(list_agents)"
rc=0
# shellcheck disable=SC2086
check_agents $agents || rc=$?
exit "$rc"
'';
}

88
nix/host-modules/swarm-archive.nix
View file

@ -22,12 +22,12 @@
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
# therefore decrypt every backup, which is the cost of pruning where clients
# cannot delete.
# - `swarm-archive-check` (weekly) runs `restic check` on every repository.
#
# The units log in as the archive's own store principal (a leaf of the store's
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
# CA, minted by ./glue-bao-tls.nix). Plain host units, no container: no
# secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); `s3` also makes the server read the S3 key as it starts.
{
pkgs,
lib,
@ -56,6 +56,7 @@ let
server = import ./lib/archive-server.nix { inherit lib; } (
{
inherit (cfg) retention;
checkSubset = cfg.checkSubset;
inherit authFile;
rclone = cfg.package;
restic = cfg.resticPackage;
@ -90,6 +91,9 @@ let
BAO_CACERT = "%d/bao-ca";
};
# The lock file the prune and the check take, so they never run on a repository together.
maintenanceLock = "/run/swarm-archive-maintenance/lock";
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
haveClientIdentity =
@ -246,6 +250,25 @@ in
example = "*-*-* 04:00:00";
description = "systemd calendar expression for the prune: the retention rules are applied then.";
};
checkSchedule = lib.mkOption {
type = lib.types.str;
default = "weekly";
example = "Sun *-*-* 03:00:00";
description = "systemd calendar expression for the integrity check of every agent's repository.";
};
checkSubset = lib.mkOption {
type = lib.types.strMatching "[0-9]+(\\.[0-9]+)?%";
default = "5%";
example = "10%";
description = ''
The share of each repository's data the integrity check reads (restic's
`--read-data-subset`). The subset is random per run, so over many runs
the checks cover a repository without any one run reading all of it. The
repository's structure is checked in full on every run whatever this is.
'';
};
};
config = lib.mkIf cfg.enable {
@ -313,6 +336,7 @@ in
"swarm-archive.service"
"swarm-archive-users.service"
"swarm-archive-prune.service"
"swarm-archive-check.service"
];
systemd.tmpfiles.rules = [
@ -418,6 +442,7 @@ in
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
@ -425,6 +450,7 @@ in
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
@ -433,6 +459,8 @@ in
Group = user;
SyslogIdentifier = "swarm-archive-prune";
CacheDirectory = "swarm-archive-prune";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
@ -454,5 +482,57 @@ in
Persistent = true;
};
};
systemd.services.swarm-archive-check = {
description = "check the integrity of every agent's restic repository";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
baoDeploy.package
cfg.resticPackage
pkgs.coreutils
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
environment =
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-check";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
Type = "oneshot";
User = user;
Group = user;
SyslogIdentifier = "swarm-archive-check";
CacheDirectory = "swarm-archive-check";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
# Covers the wait for a running prune (up to its own six hours) and the check.
TimeoutStartSec = "24h";
}
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
script = server.checkScript;
};
systemd.timers.swarm-archive-check = {
description = "check the archive's repositories";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.checkSchedule;
RandomizedDelaySec = "6h";
Persistent = true;
};
};
};
}

10
nix/host-modules/swarm-grafana.nix
View file

@ -1724,6 +1724,16 @@ in
for = "0s";
summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent";
})
# `swarm-archive-check` prints `archive check failed: <agent>`; it runs
# weekly, so the window spans a missed week.
(logCountRule {
uid = "hh-archive-check-failed";
title = "archive check failed";
windowSeconds = 700000;
expr = ''_time:8d _SYSTEMD_UNIT:="swarm-archive-check.service" _msg:"archive check failed" | stats count() as failures'';
for = "0s";
summary = "the archive's weekly integrity check failed for an agent's repository, so its backups may not restore; the unit's journal names the agent and the restic error";
})
# Logged once per detected crash, by `emit_crash_transitions`
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
# the old per-hive dashboard `agents_crashing` banner — see

163
nix/host-modules/swarm-grafana/dashboards/archive.json
View file

@ -545,8 +545,8 @@
{
"id": 9,
"type": "table",
"title": "Repository and prune per agent",
"description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
"title": "Repository, prune and check per agent",
"description": "Per agent: snapshots, size, newest snapshot, last prune and last check.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
@ -607,6 +607,36 @@
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "F",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "G",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_success\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "H",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.duration_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
}
],
"transformations": [
@ -628,7 +658,9 @@
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true
"Time 7": true,
"Time 8": true,
"Time 9": true
},
"renameByName": {
"agent": "Agent",
@ -636,7 +668,10 @@
"Value #B": "Size",
"Value #C": "Newest snapshot age",
"Value #D": "Since last prune",
"Value #E": "Prune result"
"Value #E": "Prune result",
"Value #F": "Since last check",
"Value #G": "Check result",
"Value #H": "Check duration"
}
}
}
@ -717,6 +752,64 @@
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Since last check"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
}
]
},
@ -1027,6 +1120,68 @@
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 17,
"type": "stat",
"title": "Oldest last check",
"description": "Age of the least recently checked repository. Source: check last_run_timestamp_seconds.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 18,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - min(max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d])))",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1209600
}
]
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
}
]
}