Watch
0
0
Fork
You've already forked hyperhive
0

swarm-archive: check each agent's restic repository weekly

A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
atlas 2026-11-14 11:51:25 +01:00
commit 5c30985a25

View file

@ -545,8 +545,8 @@
{
"id": 9,
"type": "table",
"title": "Repository and prune per agent",
"description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
"title": "Repository, prune and check per agent",
"description": "Per agent: snapshots, size, newest snapshot, last prune and last check.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
@ -607,6 +607,36 @@
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "F",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "G",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_success\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "H",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.duration_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
}
],
"transformations": [
@ -628,7 +658,9 @@
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true
"Time 7": true,
"Time 8": true,
"Time 9": true
},
"renameByName": {
"agent": "Agent",
@ -636,7 +668,10 @@
"Value #B": "Size",
"Value #C": "Newest snapshot age",
"Value #D": "Since last prune",
"Value #E": "Prune result"
"Value #E": "Prune result",
"Value #F": "Since last check",
"Value #G": "Check result",
"Value #H": "Check duration"
}
}
}
@ -717,6 +752,64 @@
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Since last check"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
}
]
},
@ -1027,6 +1120,68 @@
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 17,
"type": "stat",
"title": "Oldest last check",
"description": "Age of the least recently checked repository. Source: check last_run_timestamp_seconds.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 18,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - min(max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d])))",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1209600
}
]
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
}
]
}