mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-archive: check each agent's restic repository weekly

Browse source 
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas 2026-11-14 11:51:25 +01:00
commit 5c30985a25
9 changed files with 503 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/host-modules/glue-bao-readers-policy-order.nix
View file

@ -100,6 +100,10 @@ let
here = deployCfg.archive.enable;
policies = [ "swarm-bao-archive-policy" ];
};
swarm-archive-check = {
here = deployCfg.archive.enable;
policies = [ "swarm-bao-archive-policy" ];
};
# ./hive-matrix.nix, under matrix-ctl's leaf.
swarm-bao-matrix-oidc = {
here = deployCfg.matrix.enable;

1
nix/host-modules/glue-bao-readers-store-order.nix
View file

@ -72,6 +72,7 @@ let
# ./swarm-archive.nix
swarm-archive-users = deployCfg.archive.enable;
swarm-archive-prune = deployCfg.archive.enable;
swarm-archive-check = deployCfg.archive.enable;
# ./swarm-otel.nix
swarm-bao-otel-oidc =
deployCfg.swarm-otel.enable

82
nix/host-modules/lib/archive-server.nix
View file

@ -1,6 +1,6 @@
# The archive server's moving parts: the rclone command that serves the
# repositories, the script that renders its users file from the secret store,
# and the script that prunes each repository. A pure function, NOT a NixOS
# the script that prunes each repository and the one that checks them. A pure function, NOT a NixOS
# module: ../swarm-archive.nix wraps the pieces in units, and the VM test
# (nix/vm-tests/swarm-archive.nix) runs the same scripts against a stand-in
# `bao`.
@ -33,6 +33,9 @@
retention,
# The OTLP/HTTP metrics receiver's base URL (see ./otlp-gauges.nix), or null.
otlpEndpoint ? null,
# `restic check --read-data-subset=<this>`: what share of each repository's data the
# check reads, e.g. `5%`.
checkSubset ? "5%",
}:
let
atomicWriteSecret = import ./atomic-write-secret.nix { };
@ -46,6 +49,16 @@ let
s3KeyIdPath = "secret/swarm/services/archive/s3/access-key-id";
s3SecretKeyPath = "secret/swarm/services/archive/s3/secret-access-key";
# Takes the lock the prune and the check share, when the unit sets
# `ARCHIVE_MAINTENANCE_LOCK` (a path in a directory both units can write); waits
# for the holder. Held until the script exits. Needs `flock` on the unit's `path`.
maintenanceLock = ''
if [ -n "''${ARCHIVE_MAINTENANCE_LOCK:-}" ]; then
exec 9>"$ARCHIVE_MAINTENANCE_LOCK"
flock 9
fi
'';
# Slack for the clocks of the hives that stamp snapshots.
futureSkewSeconds = 3600;
@ -223,6 +236,7 @@ in
pruneScript = ''
set -euo pipefail
${storeSeededSecret}
${maintenanceLock}
# Forgets the snapshots of $repo dated after the limit, naming them in the
# log; returns non-zero when it cannot read the list or a time.
@ -311,4 +325,70 @@ in
done
exit "$failed"
'';
# `restic check` on every repository, reading `checkSubset` of its data: a random
# share each run, so repeated runs cover the repository without any run
# reading all of it. The repository's structure (index, snapshots, trees) is
# always checked in full. One repository's failure does not stop the others;
# the run exits non-zero if any failed.
checkScript = ''
set -euo pipefail
${storeSeededSecret}
${maintenanceLock}
${otlpGauges}
# Pushes the outcome of one repository's check. Never fails the run.
push_check_metrics() {
local agent="$1" ok="$2" start="$3" end="$4"
otlp_gauges swarm-archive-check "agent=$agent" \
"hyperhive.archive.check.last_run_success=$ok" \
"hyperhive.archive.check.last_run_timestamp_seconds=$end" \
"hyperhive.archive.check.duration_seconds=$(( end - start ))"
}
# Checks the repository of each agent named in the arguments; returns
# non-zero if any check failed.
check_agents() {
local agent repo start ok failed=0
for agent in "$@"; do
if [[ ! "$agent" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "archive check failed: $agent is not an agent name" >&2
failed=1
continue
fi
repo=${repoOf}
export RESTIC_PASSWORD_COMMAND="bao kv get -field=password ${resticPrefix}/$agent"
ok=0
start="$(date +%s)"
if ${restic}/bin/restic -r "$repo" --retry-lock 10m check --read-data-subset=${lib.escapeShellArg checkSubset}; then
echo "checked $agent"
ok=1
else
echo "archive check failed: $agent" >&2
failed=1
fi
push_check_metrics "$agent" "$ok" "$start" "$(date +%s)"
done
return "$failed"
}
store_init
store_login
if [ "$STORE_UP" != 1 ]; then
echo "archive check failed: could not log in to the swarm secret store" >&2
exit 1
fi
${lib.optionalString (s3 != null) s3Credentials}
list_agents() {
${listAgents}
}
# Under `set -e` a failed listing ends the run; inside the `for` it would not.
agents="$(list_agents)"
rc=0
# shellcheck disable=SC2086
check_agents $agents || rc=$?
exit "$rc"
'';
}

88
nix/host-modules/swarm-archive.nix
View file

@ -22,12 +22,12 @@
# `RESTIC_PASSWORD_COMMAND` for its `restic forget --prune`. This host can
# therefore decrypt every backup, which is the cost of pruning where clients
# cannot delete.
# - `swarm-archive-check` (weekly) runs `restic check` on every repository.
#
# The units log in as the archive's own store principal (a leaf of the store's
# CA, minted by ./glue-bao-tls.nix). Plain host units rather than a container:
# no secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); with `s3` the server also reads the S3 key from
# `swarm/services/archive/s3/…` as it starts and keeps it in rclone's environment.
# CA, minted by ./glue-bao-tls.nix). Plain host units, no container: no
# secret crosses a mount. The backend is one argument (`reposDir` or `s3`, in
# ./lib/archive-server.nix); `s3` also makes the server read the S3 key as it starts.
{
pkgs,
lib,
@ -56,6 +56,7 @@ let
server = import ./lib/archive-server.nix { inherit lib; } (
{
inherit (cfg) retention;
checkSubset = cfg.checkSubset;
inherit authFile;
rclone = cfg.package;
restic = cfg.resticPackage;
@ -90,6 +91,9 @@ let
BAO_CACERT = "%d/bao-ca";
};
# The lock file the prune and the check take, so they never run on a repository together.
maintenanceLock = "/run/swarm-archive-maintenance/lock";
serveScript = pkgs.writeShellScript "swarm-archive-serve" server.s3ServerScript;
haveClientIdentity =
@ -246,6 +250,25 @@ in
example = "*-*-* 04:00:00";
description = "systemd calendar expression for the prune: the retention rules are applied then.";
};
checkSchedule = lib.mkOption {
type = lib.types.str;
default = "weekly";
example = "Sun *-*-* 03:00:00";
description = "systemd calendar expression for the integrity check of every agent's repository.";
};
checkSubset = lib.mkOption {
type = lib.types.strMatching "[0-9]+(\\.[0-9]+)?%";
default = "5%";
example = "10%";
description = ''
The share of each repository's data the integrity check reads (restic's
`--read-data-subset`). The subset is random per run, so over many runs
the checks cover a repository without any one run reading all of it. The
repository's structure is checked in full on every run whatever this is.
'';
};
};
config = lib.mkIf cfg.enable {
@ -313,6 +336,7 @@ in
"swarm-archive.service"
"swarm-archive-users.service"
"swarm-archive-prune.service"
"swarm-archive-check.service"
];
systemd.tmpfiles.rules = [
@ -418,6 +442,7 @@ in
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
@ -425,6 +450,7 @@ in
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-prune";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
@ -433,6 +459,8 @@ in
Group = user;
SyslogIdentifier = "swarm-archive-prune";
CacheDirectory = "swarm-archive-prune";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
@ -454,5 +482,57 @@ in
Persistent = true;
};
};
systemd.services.swarm-archive-check = {
description = "check the integrity of every agent's restic repository";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
path = [
baoDeploy.package
cfg.resticPackage
pkgs.coreutils
pkgs.curl
pkgs.gnugrep
pkgs.jq
pkgs.util-linux
]
# restic's rclone backend starts rclone.
++ lib.optional useS3 cfg.package;
environment =
ownerStoreEnvironment
// {
RESTIC_CACHE_DIR = "%C/swarm-archive-check";
ARCHIVE_MAINTENANCE_LOCK = maintenanceLock;
}
// lib.optionalAttrs useS3 ({ RCLONE_CONFIG = "/dev/null"; } // server.environment);
serviceConfig = {
Type = "oneshot";
User = user;
Group = user;
SyslogIdentifier = "swarm-archive-check";
CacheDirectory = "swarm-archive-check";
RuntimeDirectory = "swarm-archive-maintenance";
RuntimeDirectoryPreserve = "yes";
LoadCredential = ownerStoreCredentials;
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
PrivateTmp = true;
PrivateDevices = true;
# Covers the wait for a running prune (up to its own six hours) and the check.
TimeoutStartSec = "24h";
}
// lib.optionalAttrs (!useS3) { ReadWritePaths = [ reposDir ]; };
script = server.checkScript;
};
systemd.timers.swarm-archive-check = {
description = "check the archive's repositories";
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = cfg.checkSchedule;
RandomizedDelaySec = "6h";
Persistent = true;
};
};
};
}

10
nix/host-modules/swarm-grafana.nix
View file

@ -1724,6 +1724,16 @@ in
for = "0s";
summary = "the archive's daily prune failed for an agent's repository, so its retention is not applied; the unit's journal names the agent";
})
# `swarm-archive-check` prints `archive check failed: <agent>`; it runs
# weekly, so the window spans a missed week.
(logCountRule {
uid = "hh-archive-check-failed";
title = "archive check failed";
windowSeconds = 700000;
expr = ''_time:8d _SYSTEMD_UNIT:="swarm-archive-check.service" _msg:"archive check failed" | stats count() as failures'';
for = "0s";
summary = "the archive's weekly integrity check failed for an agent's repository, so its backups may not restore; the unit's journal names the agent and the restic error";
})
# Logged once per detected crash, by `emit_crash_transitions`
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
# the old per-hive dashboard `agents_crashing` banner — see

163
nix/host-modules/swarm-grafana/dashboards/archive.json
View file

@ -545,8 +545,8 @@
{
"id": 9,
"type": "table",
"title": "Repository and prune per agent",
"description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
"title": "Repository, prune and check per agent",
"description": "Per agent: snapshots, size, newest snapshot, last prune and last check.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
@ -607,6 +607,36 @@
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "F",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "G",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_success\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "H",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.duration_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
}
],
"transformations": [
@ -628,7 +658,9 @@
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true
"Time 7": true,
"Time 8": true,
"Time 9": true
},
"renameByName": {
"agent": "Agent",
@ -636,7 +668,10 @@
"Value #B": "Size",
"Value #C": "Newest snapshot age",
"Value #D": "Since last prune",
"Value #E": "Prune result"
"Value #E": "Prune result",
"Value #F": "Since last check",
"Value #G": "Check result",
"Value #H": "Check duration"
}
}
}
@ -717,6 +752,64 @@
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Since last check"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
}
]
},
@ -1027,6 +1120,68 @@
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 17,
"type": "stat",
"title": "Oldest last check",
"description": "Age of the least recently checked repository. Source: check last_run_timestamp_seconds.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 18,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - min(max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d])))",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1209600
}
]
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
}
]
}

9
nix/module-eval/builder.nix
View file

@ -617,6 +617,7 @@ let
a = atticHere.systemd.services.swarm-archive;
users = atticHere.systemd.services.swarm-archive-users;
prune = atticHere.systemd.services.swarm-archive-prune;
check = atticHere.systemd.services.swarm-archive-check;
vhost = atticHere.services.hyperhive.gateway.vhosts."archive.t.local";
cfg = atticHere.services.hyperhive.deploy.archive;
names = m: m.services.hyperhive.swarm.serviceDomains;
@ -642,6 +643,14 @@ let
&& !(lib.hasInfix "password" (lib.toLower (builtins.toJSON a.environment)))
&& atticHere.systemd.timers ? swarm-archive-users
&& atticHere.systemd.timers.swarm-archive-prune.timerConfig.OnCalendar == "daily"
&& atticHere.systemd.timers.swarm-archive-check.timerConfig.OnCalendar == "weekly"
&& atticHere.systemd.timers.swarm-archive-check.timerConfig.Persistent
&& lib.hasInfix "--read-data-subset=5%" check.script
&& check.serviceConfig.User == "swarm-archive"
&& check.environment.ARCHIVE_MAINTENANCE_LOCK == prune.environment.ARCHIVE_MAINTENANCE_LOCK
&& check.serviceConfig.RuntimeDirectory == prune.serviceConfig.RuntimeDirectory
&& lib.hasPrefix "/run/${check.serviceConfig.RuntimeDirectory}/" check.environment.ARCHIVE_MAINTENANCE_LOCK
&& lib.elem "swarm-bao-archive-policy.service" check.after
&& lib.elem "swarm-bao-archive-policy.service" users.after
&& !(bare.systemd.services ? swarm-archive)
&& !(bare.systemd.services ? swarm-archive-users);

31
nix/script-tests/archive-metrics.nix
View file

@ -1,6 +1,6 @@
# `checks.script-test-archive-metrics` — runs the metric functions of the archive's
# backup and prune scripts (../host-modules/lib/archive-backup.nix and
# archive-server.nix) against a stub `curl` and a fake `restic`. The cases are in
# archive-server.nix, including the check script's loop) against a stub `curl` and a fake `restic`. The cases are in
# ./archive-metrics.sh, which cuts each function out of the rendered script.
{
pkgs,
@ -11,6 +11,18 @@ let
# Answers from the files named by FAKE_SNAPSHOTS and FAKE_STATS; an unset
# variable makes that subcommand fail, and so does a call without --no-lock:
# the metrics only read, and must not wait on a lock.
# `check` is the integrity check: it must hold the repository lock, and fails
# for the repositories named in FAKE_CHECK_FAIL (space separated agents).
case " $* " in
*" check "*)
echo "check $*" >>"$FAKE_LOG"
case " $* " in *" --no-lock "*) exit 1 ;; esac
for a in ''${FAKE_CHECK_FAIL:-}; do
case " $* " in *"/$a/ "*) exit 1 ;; esac
done
exit 0
;;
esac
case " $* " in *" --no-lock "*) ;; *) exit 1 ;; esac
case " $* " in
*" snapshots "*) [ -n "''${FAKE_SNAPSHOTS:-}" ] && cat "$FAKE_SNAPSHOTS" ;;
@ -25,6 +37,7 @@ pkgs.runCommand "hyperhive-script-test-archive-metrics"
nativeBuildInputs = [
pkgs.coreutils
pkgs.jq
pkgs.util-linux
];
BACKUP_SCRIPT = pkgs.writeText "archive-backup.sh" (
import ../host-modules/lib/archive-backup.nix { inherit (pkgs) lib; } {
@ -39,6 +52,22 @@ pkgs.runCommand "hyperhive-script-test-archive-metrics"
otlpEndpoint = endpoint;
}
);
CHECK_SCRIPT = pkgs.writeText "archive-check.sh" (
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
rclone = pkgs.rclone;
restic = fakeRestic;
reposDir = "/repos";
addr = "127.0.0.1:1";
authFile = "/auth";
authGroup = "g";
retention = {
keepWithin = null;
keepLast = 1;
};
checkSubset = "7%";
otlpEndpoint = endpoint;
}).checkScript
);
PRUNE_SCRIPT = pkgs.writeText "archive-prune.sh" (
(import ../host-modules/lib/archive-server.nix { inherit (pkgs) lib; } {
rclone = pkgs.rclone;

96
nix/script-tests/archive-metrics.sh
View file

@ -16,6 +16,8 @@ mkdir -p "$root/bin"
cat >"$root/bin/curl" <<STUB
#!$BASH
cat >"$root/curl.payload"
cat "$root/curl.payload" >>"$root/curl.all"
echo >>"$root/curl.all"
STUB
chmod +x "$root/bin/curl"
@ -185,5 +187,99 @@ else
fail "failed: $(cat "$root/err")"
fi
# --- the check side ---
c=hyperhive.archive.check
{
cut_function "$CHECK_SCRIPT" otlp_gauges
cut_function "$CHECK_SCRIPT" push_check_metrics
cut_function "$CHECK_SCRIPT" check_agents
} >"$root/check-fns.sh"
grep -q 'check_agents()' "$root/check-fns.sh" || { echo "FAILED: check_agents not found"; exit 1; }
grep -q 'push_check_metrics()' "$root/check-fns.sh" || { echo "FAILED: push_check_metrics not found"; exit 1; }
# The gauges pushed for agent $1 in the last run, as `name=value` lines.
agent_gauges() {
jq -rs --arg a "$1" '.[] | select(.resourceMetrics[0].scopeMetrics[0].metrics[0].gauge.dataPoints[0].attributes[0].value.stringValue == $a)
| .resourceMetrics[0].scopeMetrics[0].metrics[] | "\(.name)=\(.gauge.dataPoints[0].asDouble)"' "$root/curl.all"
}
case="the check loop goes on after a failure, exits non-zero and names the agent that failed"
rm -f "$root/curl.all" "$root/fake.log"
: >"$root/fake.log"
FAKE_LOG="$root/fake.log" FAKE_CHECK_FAIL=beta run "$root/check-fns.sh" check_agents alpha beta gamma
rc=$?
[ "$rc" != 0 ] || fail "exit status was 0"
grep -qx 'archive check failed: beta' "$root/err" || fail "no failure line: $(cat "$root/err")"
grep -q 'archive check failed: alpha\|archive check failed: gamma' "$root/err" && fail "a passing agent was reported: $(cat "$root/err")"
[ "$(wc -l <"$root/fake.log")" = 3 ] || fail "want a check of every agent: $(cat "$root/fake.log")"
grep -q '/repos/gamma/ .*check --read-data-subset=7%' "$root/fake.log" || fail "gamma not checked with the subset: $(cat "$root/fake.log")"
grep -qx 'checked alpha' "$root/out" || fail "no success line: $(cat "$root/out")"
case="the check pushes each agent's own result"
[ "$(agent_gauges alpha | grep -c .)" = 3 ] || fail "alpha gauges: $(agent_gauges alpha)"
agent_gauges alpha | grep -qx "$c.last_run_success=1" || fail "alpha result: $(agent_gauges alpha)"
agent_gauges alpha | grep -q "^$c.last_run_timestamp_seconds=[0-9]" || fail "alpha time: $(agent_gauges alpha)"
agent_gauges alpha | grep -q "^$c.duration_seconds=[0-9]" || fail "alpha duration: $(agent_gauges alpha)"
agent_gauges beta | grep -qx "$c.last_run_success=0" || fail "beta result: $(agent_gauges beta)"
agent_gauges gamma | grep -qx "$c.last_run_success=1" || fail "gamma result: $(agent_gauges gamma)"
[ "$(jq -rs '.[0].resourceMetrics[0].resource.attributes[0].value.stringValue' "$root/curl.all")" = swarm-archive-check ] ||
fail "service.name"
case="the check pushes the duration it measured"
if run "$root/check-fns.sh" push_check_metrics alpha 1 100 190; then
want_gauges "$c.last_run_success=1
$c.last_run_timestamp_seconds=190
$c.duration_seconds=90"
else
fail "failed: $(cat "$root/err")"
fi
case="the check exits zero when every repository passes"
if FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents alpha beta; then
grep -q 'archive check failed' "$root/err" && fail "failure line: $(cat "$root/err")"
else
fail "exit status was non-zero: $(cat "$root/err")"
fi
case="the check refuses a name that is not an agent name and goes on"
: >"$root/fake.log"
FAKE_LOG="$root/fake.log" run "$root/check-fns.sh" check_agents 'bad name;x' alpha
rc=$?
[ "$rc" != 0 ] || fail "exit status was 0"
grep -qx 'archive check failed: bad name;x is not an agent name' "$root/err" || fail "no refusal: $(cat "$root/err")"
[ "$(wc -l <"$root/fake.log")" = 1 ] || fail "alpha not checked: $(cat "$root/fake.log")"
# --- the lock the prune and the check share ---
lock_prologue() {
awk '/^if \[ -n "\$\{ARCHIVE_MAINTENANCE_LOCK:-\}" \]; then$/ {on=1} on {print} on && /^fi$/ {exit}' "$1"
}
for s in CHECK_SCRIPT PRUNE_SCRIPT; do
lock_prologue "${!s}" >"$root/lock-$s.sh"
grep -q flock "$root/lock-$s.sh" || { echo "FAILED: $s has no maintenance lock"; exit 1; }
done
case="the check waits while the prune holds the lock, and runs once it is free"
lockfile="$root/maintenance.lock"
flock "$lockfile" sleep 5 &
holder=$!
sleep 0.5
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-CHECK_SCRIPT.sh"
[ "$?" = 124 ] || fail "did not wait for the held lock"
kill "$holder" 2>/dev/null
wait "$holder" 2>/dev/null
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "did not run on a free lock"
case="the prune waits on the same lock"
flock "$lockfile" sleep 5 &
holder=$!
sleep 0.5
ARCHIVE_MAINTENANCE_LOCK="$lockfile" timeout 1 bash "$root/lock-PRUNE_SCRIPT.sh"
[ "$?" = 124 ] || fail "did not wait for the held lock"
kill "$holder" 2>/dev/null
wait "$holder" 2>/dev/null
case="without the lock variable the scripts take no lock"
timeout 5 bash "$root/lock-CHECK_SCRIPT.sh" || fail "failed without the variable"
[ "$failures" -eq 0 ] || exit 1
echo "all cases passed"