mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/host-modules/swarm-grafana/dashboards/archive.json
atlas 5c30985a25 swarm-archive: check each agent's restic repository weekly 
A weekly swarm-archive-check unit runs restic check --read-data-subset on every repository on the archive host, serialised with the prune by a shared flock. It logs archive check failed per agent, pushes check gauges, and has an alert and dashboard columns.
2026-11-14 11:53:27 +01:00

1187 lines
30 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"title": "hyperhive · backups (restic)",
"uid": "hyperhive-archive",
"description": "Each agent's restic backup to the swarm archive, and the archive's prune.",
"editable": true,
"refresh": "1m",
"schemaVersion": 39,
"tags": ["hyperhive", "swarm", "archive", "backup", "restic"],
"time": {
"from": "now-24h",
"to": "now"
},
"timezone": "utc",
"version": 1,
"templating": {
"list": [
{
"name": "hive",
"label": "Hive",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\"}, hive)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "agent",
"label": "Agent",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\"}, agent)",
"refresh": 2,
"includeAll": true,
"allValue": ".*",
"multi": true,
"current": {
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Backups",
"collapsed": false,
"gridPos": {
"x": 0,
"y": 0,
"w": 24,
"h": 1
},
"panels": []
},
{
"id": 2,
"type": "stat",
"title": "Agents past 2× the interval",
"description": "Agents whose last successful backup is over 2 hours old, twice the hourly schedule.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 0,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) > 7200) or vector(0)",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "red"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 3,
"type": "stat",
"title": "Agents whose last run failed",
"description": "Agents whose latest backup run ended in failure. Source: last_run_success gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 6,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) == 0) or vector(0)",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "red"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 4,
"type": "stat",
"title": "Agents backed up",
"description": "Agents that reported a backup run in the last 7 days. Source: last_run_success gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 12,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))) or vector(0)",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "fixed",
"fixedColor": "green"
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
},
{
"id": 5,
"type": "table",
"title": "Last backup per agent",
"description": "Per agent: time since last success (red over 2 hours), last result, duration, data added.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 0,
"y": 5,
"w": 24,
"h": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "D",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
}
],
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "agent",
"mode": "outer"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true
},
"renameByName": {
"agent": "Agent",
"Value #A": "Since last success",
"Value #B": "Last result",
"Value #C": "Duration",
"Value #D": "Data added"
}
}
}
],
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto"
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Since last success"
},
"properties": [
{
"id": "unit",
"value": "s"
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
},
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 7200
}
]
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Last result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Data added"
},
"properties": [
{
"id": "unit",
"value": "bytes"
}
]
}
]
},
"options": {
"showHeader": true,
"sortBy": [
{
"displayName": "Since last success",
"desc": true
}
]
}
},
{
"id": 6,
"type": "timeseries",
"title": "Data added per run",
"description": "Bytes each backup run added to the repository, per agent. Source: data_added_bytes gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 0,
"y": 14,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 7,
"type": "timeseries",
"title": "Backup duration",
"description": "Wall time of each agent's backup run, snapshot included. Source: duration_seconds gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 12,
"y": 14,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 8,
"type": "row",
"title": "Repositories",
"collapsed": false,
"gridPos": {
"x": 0,
"y": 22,
"w": 24,
"h": 1
},
"panels": []
},
{
"id": 9,
"type": "table",
"title": "Repository, prune and check per agent",
"description": "Per agent: snapshots, size, newest snapshot, last prune and last check.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 0,
"y": 23,
"w": 24,
"h": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "B",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "C",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.latest_snapshot_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "D",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "E",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
"instant": true,
"format": "table"
},
{
"refId": "F",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "G",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_success\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
},
{
"refId": "H",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.duration_seconds\", agent=~\"$agent\"}[30d]))",
"instant": true,
"format": "table"
}
],
"transformations": [
{
"id": "joinByField",
"options": {
"byField": "agent",
"mode": "outer"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Time 1": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true,
"Time 8": true,
"Time 9": true
},
"renameByName": {
"agent": "Agent",
"Value #A": "Snapshots",
"Value #B": "Size",
"Value #C": "Newest snapshot age",
"Value #D": "Since last prune",
"Value #E": "Prune result",
"Value #F": "Since last check",
"Value #G": "Check result",
"Value #H": "Check duration"
}
}
}
],
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto"
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Size"
},
"properties": [
{
"id": "unit",
"value": "bytes"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Newest snapshot age"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Since last prune"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Prune result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "Since last check"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Check result"
},
"properties": [
{
"id": "mappings",
"value": [
{
"type": "value",
"options": {
"0": {
"text": "failed",
"color": "red",
"index": 0
},
"1": {
"text": "ok",
"color": "green",
"index": 1
}
}
}
]
},
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
}
]
},
"options": {
"showHeader": true,
"sortBy": [
{
"displayName": "Agent",
"desc": true
}
]
}
},
{
"id": 10,
"type": "timeseries",
"title": "Snapshots per repository",
"description": "Snapshots each agent's repository holds after the prune. Source: repo.snapshots gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 11,
"type": "timeseries",
"title": "Repository size",
"description": "Size of the data in each agent's repository. Source: repo.size_bytes gauge.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"})",
"legendFormat": "{{agent}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes",
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 12,
"type": "row",
"title": "Failures",
"collapsed": false,
"gridPos": {
"x": 0,
"y": 40,
"w": 24,
"h": 1
},
"panels": []
},
{
"id": 13,
"type": "timeseries",
"title": "Failures over time",
"description": "Backup and prune failure lines per interval.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"x": 0,
"y": 41,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\" | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "backup failed"
},
{
"refId": "B",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\" | stats count_uniq(__CURSOR) as n",
"queryType": "statsRange",
"legendFormat": "prune failed"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "bars",
"lineWidth": 1,
"fillOpacity": 60,
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true,
"calcs": []
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
}
},
{
"id": 14,
"type": "logs",
"title": "Recent backup failures",
"description": "Agents hive-archive-backup could not back up, newest first.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"x": 12,
"y": 41,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\"",
"queryType": "instant"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 15,
"type": "logs",
"title": "Recent prune failures",
"description": "Repositories swarm-archive-prune could not prune, newest first.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"x": 0,
"y": 49,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\"",
"queryType": "instant"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 16,
"type": "logs",
"title": "Recent backup runs",
"description": "Agents backed up, skipped and failed by hive-archive-backup, newest first.",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"gridPos": {
"x": 12,
"y": 49,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "victoriametrics-logs-datasource",
"uid": "@logsDatasourceUid@"
},
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\"",
"queryType": "instant"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
}
},
{
"id": 17,
"type": "stat",
"title": "Oldest last check",
"description": "Age of the least recently checked repository. Source: check last_run_timestamp_seconds.",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"gridPos": {
"x": 18,
"y": 1,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "@datasourceUid@"
},
"expr": "time() - min(max by (agent) (last_over_time({__name__=\"hyperhive.archive.check.last_run_timestamp_seconds\", agent=~\"$agent\"}[30d])))",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"decimals": 0,
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1209600
}
]
},
"mappings": []
},
"overrides": []
},
"options": {
"graphMode": "none",
"colorMode": "value",
"textMode": "value",
"justifyMode": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
}
}
]
}
View git blame Copy permalink