swarm-archive: check each agent's restic repository weekly
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
This commit is contained in:
parent
c4919c5945
commit
5c30985a25
1 changed files with 503 additions and 18 deletions
|
|
@ -545,8 +545,8 @@
|
|||
{
|
||||
"id": 9,
|
||||
"type": "table",
|
||||
"title": "Repository and prune per agent",
|
||||
"description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
|
||||
"title": "Repository, prune and check per agent",
|
||||
"description": "Per agent: snapshots, size, newest snapshot, last prune and last check.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
|
|
@ -607,6 +607,36 @@
|
|||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "F",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d]))",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "G",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_success\", agent=~\"$agent\"}[30d]))",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "H",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.duration_seconds\", agent=~\"$agent\"}[30d]))",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
|
|
@ -628,7 +658,9 @@
|
|||
"Time 4": true,
|
||||
"Time 5": true,
|
||||
"Time 6": true,
|
||||
"Time 7": true
|
||||
"Time 7": true,
|
||||
"Time 8": true,
|
||||
"Time 9": true
|
||||
},
|
||||
"renameByName": {
|
||||
"agent": "Agent",
|
||||
|
|
@ -636,7 +668,10 @@
|
|||
"Value #B": "Size",
|
||||
"Value #C": "Newest snapshot age",
|
||||
"Value #D": "Since last prune",
|
||||
"Value #E": "Prune result"
|
||||
"Value #E": "Prune result",
|
||||
"Value #F": "Since last check",
|
||||
"Value #G": "Check result",
|
||||
"Value #H": "Check duration"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -717,6 +752,64 @@
|
|||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Since last check"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Check duration"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Check result"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "mappings",
|
||||
"value": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"0": {
|
||||
"text": "failed",
|
||||
"color": "red",
|
||||
"index": 0
|
||||
},
|
||||
"1": {
|
||||
"text": "ok",
|
||||
"color": "green",
|
||||
"index": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "custom.cellOptions",
|
||||
"value": {
|
||||
"type": "color-text"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -1027,6 +1120,68 @@
|
|||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "stat",
|
||||
"title": "Oldest last check",
|
||||
"description": "Age of the least recently checked repository. Source: check last_run_timestamp_seconds.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 18,
|
||||
"y": 1,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "time() - min(max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d])))",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1209600
|
||||
}
|
||||
]
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue