1032 lines
26 KiB
JSON
1032 lines
26 KiB
JSON
{
|
||
"title": "hyperhive · backups (restic)",
|
||
"uid": "hyperhive-archive",
|
||
"description": "Each agent's restic backup to the swarm archive, and the archive's prune.",
|
||
"editable": true,
|
||
"refresh": "1m",
|
||
"schemaVersion": 39,
|
||
"tags": ["hyperhive", "swarm", "archive", "backup", "restic"],
|
||
"time": {
|
||
"from": "now-24h",
|
||
"to": "now"
|
||
},
|
||
"timezone": "utc",
|
||
"version": 1,
|
||
"templating": {
|
||
"list": [
|
||
{
|
||
"name": "hive",
|
||
"label": "Hive",
|
||
"type": "query",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\"}, hive)",
|
||
"refresh": 2,
|
||
"includeAll": true,
|
||
"allValue": ".*",
|
||
"multi": true,
|
||
"current": {
|
||
"text": "All",
|
||
"value": "$__all"
|
||
},
|
||
"sort": 1
|
||
},
|
||
{
|
||
"name": "agent",
|
||
"label": "Agent",
|
||
"type": "query",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\"}, agent)",
|
||
"refresh": 2,
|
||
"includeAll": true,
|
||
"allValue": ".*",
|
||
"multi": true,
|
||
"current": {
|
||
"text": "All",
|
||
"value": "$__all"
|
||
},
|
||
"sort": 1
|
||
}
|
||
]
|
||
},
|
||
"panels": [
|
||
{
|
||
"id": 1,
|
||
"type": "row",
|
||
"title": "Backups",
|
||
"collapsed": false,
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 0,
|
||
"w": 24,
|
||
"h": 1
|
||
},
|
||
"panels": []
|
||
},
|
||
{
|
||
"id": 2,
|
||
"type": "stat",
|
||
"title": "Agents past 2× the interval",
|
||
"description": "Agents whose last successful backup is over 2 hours old, twice the hourly schedule.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 1,
|
||
"w": 6,
|
||
"h": 4
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "count(time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) > 7200) or vector(0)",
|
||
"instant": true
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "short",
|
||
"decimals": 0,
|
||
"color": {
|
||
"mode": "fixed",
|
||
"fixedColor": "red"
|
||
},
|
||
"mappings": []
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"graphMode": "none",
|
||
"colorMode": "value",
|
||
"textMode": "value",
|
||
"justifyMode": "auto",
|
||
"reduceOptions": {
|
||
"calcs": ["lastNotNull"],
|
||
"fields": "",
|
||
"values": false
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 3,
|
||
"type": "stat",
|
||
"title": "Agents whose last run failed",
|
||
"description": "Agents whose latest backup run ended in failure. Source: last_run_success gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 6,
|
||
"y": 1,
|
||
"w": 6,
|
||
"h": 4
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) == 0) or vector(0)",
|
||
"instant": true
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "short",
|
||
"decimals": 0,
|
||
"color": {
|
||
"mode": "fixed",
|
||
"fixedColor": "red"
|
||
},
|
||
"mappings": []
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"graphMode": "none",
|
||
"colorMode": "value",
|
||
"textMode": "value",
|
||
"justifyMode": "auto",
|
||
"reduceOptions": {
|
||
"calcs": ["lastNotNull"],
|
||
"fields": "",
|
||
"values": false
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 4,
|
||
"type": "stat",
|
||
"title": "Agents backed up",
|
||
"description": "Agents that reported a backup run in the last 7 days. Source: last_run_success gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 12,
|
||
"y": 1,
|
||
"w": 6,
|
||
"h": 4
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))) or vector(0)",
|
||
"instant": true
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "short",
|
||
"decimals": 0,
|
||
"color": {
|
||
"mode": "fixed",
|
||
"fixedColor": "green"
|
||
},
|
||
"mappings": []
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"graphMode": "none",
|
||
"colorMode": "value",
|
||
"textMode": "value",
|
||
"justifyMode": "auto",
|
||
"reduceOptions": {
|
||
"calcs": ["lastNotNull"],
|
||
"fields": "",
|
||
"values": false
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 5,
|
||
"type": "table",
|
||
"title": "Last backup per agent",
|
||
"description": "Per agent: time since last success (red over 2 hours), last result, duration, data added.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 5,
|
||
"w": 24,
|
||
"h": 9
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "B",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "C",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "D",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
}
|
||
],
|
||
"transformations": [
|
||
{
|
||
"id": "joinByField",
|
||
"options": {
|
||
"byField": "agent",
|
||
"mode": "outer"
|
||
}
|
||
},
|
||
{
|
||
"id": "organize",
|
||
"options": {
|
||
"excludeByName": {
|
||
"Time": true,
|
||
"Time 1": true,
|
||
"Time 2": true,
|
||
"Time 3": true,
|
||
"Time 4": true,
|
||
"Time 5": true,
|
||
"Time 6": true,
|
||
"Time 7": true
|
||
},
|
||
"renameByName": {
|
||
"agent": "Agent",
|
||
"Value #A": "Since last success",
|
||
"Value #B": "Last result",
|
||
"Value #C": "Duration",
|
||
"Value #D": "Data added"
|
||
}
|
||
}
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"align": "auto"
|
||
}
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Since last success"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "s"
|
||
},
|
||
{
|
||
"id": "custom.cellOptions",
|
||
"value": {
|
||
"type": "color-text"
|
||
}
|
||
},
|
||
{
|
||
"id": "thresholds",
|
||
"value": {
|
||
"mode": "absolute",
|
||
"steps": [
|
||
{
|
||
"color": "green",
|
||
"value": null
|
||
},
|
||
{
|
||
"color": "red",
|
||
"value": 7200
|
||
}
|
||
]
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Last result"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "mappings",
|
||
"value": [
|
||
{
|
||
"type": "value",
|
||
"options": {
|
||
"0": {
|
||
"text": "failed",
|
||
"color": "red",
|
||
"index": 0
|
||
},
|
||
"1": {
|
||
"text": "ok",
|
||
"color": "green",
|
||
"index": 1
|
||
}
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"id": "custom.cellOptions",
|
||
"value": {
|
||
"type": "color-text"
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Duration"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "s"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Data added"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "bytes"
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"options": {
|
||
"showHeader": true,
|
||
"sortBy": [
|
||
{
|
||
"displayName": "Since last success",
|
||
"desc": true
|
||
}
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"id": 6,
|
||
"type": "timeseries",
|
||
"title": "Data added per run",
|
||
"description": "Bytes each backup run added to the repository, per agent. Source: data_added_bytes gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 14,
|
||
"w": 12,
|
||
"h": 8
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||
"legendFormat": "{{agent}}"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "bytes",
|
||
"color": {
|
||
"mode": "palette-classic"
|
||
},
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"lineWidth": 1,
|
||
"fillOpacity": 10,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"axisSoftMin": 0
|
||
}
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "list",
|
||
"placement": "bottom",
|
||
"showLegend": true,
|
||
"calcs": []
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 7,
|
||
"type": "timeseries",
|
||
"title": "Backup duration",
|
||
"description": "Wall time of each agent's backup run, snapshot included. Source: duration_seconds gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 12,
|
||
"y": 14,
|
||
"w": 12,
|
||
"h": 8
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"})",
|
||
"legendFormat": "{{agent}}"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "s",
|
||
"color": {
|
||
"mode": "palette-classic"
|
||
},
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"lineWidth": 1,
|
||
"fillOpacity": 10,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"axisSoftMin": 0
|
||
}
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "list",
|
||
"placement": "bottom",
|
||
"showLegend": true,
|
||
"calcs": []
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 8,
|
||
"type": "row",
|
||
"title": "Repositories",
|
||
"collapsed": false,
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 22,
|
||
"w": 24,
|
||
"h": 1
|
||
},
|
||
"panels": []
|
||
},
|
||
{
|
||
"id": 9,
|
||
"type": "table",
|
||
"title": "Repository and prune per agent",
|
||
"description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 23,
|
||
"w": 24,
|
||
"h": 9
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "B",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "C",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.latest_snapshot_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "D",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
},
|
||
{
|
||
"refId": "E",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
|
||
"instant": true,
|
||
"format": "table"
|
||
}
|
||
],
|
||
"transformations": [
|
||
{
|
||
"id": "joinByField",
|
||
"options": {
|
||
"byField": "agent",
|
||
"mode": "outer"
|
||
}
|
||
},
|
||
{
|
||
"id": "organize",
|
||
"options": {
|
||
"excludeByName": {
|
||
"Time": true,
|
||
"Time 1": true,
|
||
"Time 2": true,
|
||
"Time 3": true,
|
||
"Time 4": true,
|
||
"Time 5": true,
|
||
"Time 6": true,
|
||
"Time 7": true
|
||
},
|
||
"renameByName": {
|
||
"agent": "Agent",
|
||
"Value #A": "Snapshots",
|
||
"Value #B": "Size",
|
||
"Value #C": "Newest snapshot age",
|
||
"Value #D": "Since last prune",
|
||
"Value #E": "Prune result"
|
||
}
|
||
}
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"custom": {
|
||
"align": "auto"
|
||
}
|
||
},
|
||
"overrides": [
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Size"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "bytes"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Newest snapshot age"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "s"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Since last prune"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "unit",
|
||
"value": "s"
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"matcher": {
|
||
"id": "byName",
|
||
"options": "Prune result"
|
||
},
|
||
"properties": [
|
||
{
|
||
"id": "mappings",
|
||
"value": [
|
||
{
|
||
"type": "value",
|
||
"options": {
|
||
"0": {
|
||
"text": "failed",
|
||
"color": "red",
|
||
"index": 0
|
||
},
|
||
"1": {
|
||
"text": "ok",
|
||
"color": "green",
|
||
"index": 1
|
||
}
|
||
}
|
||
}
|
||
]
|
||
},
|
||
{
|
||
"id": "custom.cellOptions",
|
||
"value": {
|
||
"type": "color-text"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
},
|
||
"options": {
|
||
"showHeader": true,
|
||
"sortBy": [
|
||
{
|
||
"displayName": "Agent",
|
||
"desc": true
|
||
}
|
||
]
|
||
}
|
||
},
|
||
{
|
||
"id": 10,
|
||
"type": "timeseries",
|
||
"title": "Snapshots per repository",
|
||
"description": "Snapshots each agent's repository holds after the prune. Source: repo.snapshots gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 32,
|
||
"w": 12,
|
||
"h": 8
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"})",
|
||
"legendFormat": "{{agent}}"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "short",
|
||
"color": {
|
||
"mode": "palette-classic"
|
||
},
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"lineWidth": 1,
|
||
"fillOpacity": 10,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"axisSoftMin": 0
|
||
}
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "list",
|
||
"placement": "bottom",
|
||
"showLegend": true,
|
||
"calcs": []
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 11,
|
||
"type": "timeseries",
|
||
"title": "Repository size",
|
||
"description": "Size of the data in each agent's repository. Source: repo.size_bytes gauge.",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 12,
|
||
"y": 32,
|
||
"w": 12,
|
||
"h": 8
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "prometheus",
|
||
"uid": "@datasourceUid@"
|
||
},
|
||
"expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"})",
|
||
"legendFormat": "{{agent}}"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "bytes",
|
||
"color": {
|
||
"mode": "palette-classic"
|
||
},
|
||
"custom": {
|
||
"drawStyle": "line",
|
||
"lineWidth": 1,
|
||
"fillOpacity": 10,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"axisSoftMin": 0
|
||
}
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "list",
|
||
"placement": "bottom",
|
||
"showLegend": true,
|
||
"calcs": []
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 12,
|
||
"type": "row",
|
||
"title": "Failures",
|
||
"collapsed": false,
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 40,
|
||
"w": 24,
|
||
"h": 1
|
||
},
|
||
"panels": []
|
||
},
|
||
{
|
||
"id": 13,
|
||
"type": "timeseries",
|
||
"title": "Failures over time",
|
||
"description": "Backup and prune failure lines per interval.",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 41,
|
||
"w": 12,
|
||
"h": 8
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\" | stats count_uniq(__CURSOR) as n",
|
||
"queryType": "statsRange",
|
||
"legendFormat": "backup failed"
|
||
},
|
||
{
|
||
"refId": "B",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\" | stats count_uniq(__CURSOR) as n",
|
||
"queryType": "statsRange",
|
||
"legendFormat": "prune failed"
|
||
}
|
||
],
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"unit": "short",
|
||
"decimals": 0,
|
||
"color": {
|
||
"mode": "palette-classic"
|
||
},
|
||
"custom": {
|
||
"drawStyle": "bars",
|
||
"lineWidth": 1,
|
||
"fillOpacity": 60,
|
||
"showPoints": "never",
|
||
"spanNulls": false,
|
||
"axisSoftMin": 0
|
||
}
|
||
},
|
||
"overrides": []
|
||
},
|
||
"options": {
|
||
"legend": {
|
||
"displayMode": "list",
|
||
"placement": "bottom",
|
||
"showLegend": true,
|
||
"calcs": []
|
||
},
|
||
"tooltip": {
|
||
"mode": "multi",
|
||
"sort": "desc"
|
||
}
|
||
}
|
||
},
|
||
{
|
||
"id": 14,
|
||
"type": "logs",
|
||
"title": "Recent backup failures",
|
||
"description": "Agents hive-archive-backup could not back up, newest first.",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 12,
|
||
"y": 41,
|
||
"w": 12,
|
||
"h": 9
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\"",
|
||
"queryType": "instant"
|
||
}
|
||
],
|
||
"options": {
|
||
"showTime": true,
|
||
"wrapLogMessage": true,
|
||
"prettifyLogMessage": false,
|
||
"enableLogDetails": true,
|
||
"sortOrder": "Descending",
|
||
"dedupStrategy": "none"
|
||
}
|
||
},
|
||
{
|
||
"id": 15,
|
||
"type": "logs",
|
||
"title": "Recent prune failures",
|
||
"description": "Repositories swarm-archive-prune could not prune, newest first.",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 0,
|
||
"y": 49,
|
||
"w": 12,
|
||
"h": 9
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\"",
|
||
"queryType": "instant"
|
||
}
|
||
],
|
||
"options": {
|
||
"showTime": true,
|
||
"wrapLogMessage": true,
|
||
"prettifyLogMessage": false,
|
||
"enableLogDetails": true,
|
||
"sortOrder": "Descending",
|
||
"dedupStrategy": "none"
|
||
}
|
||
},
|
||
{
|
||
"id": 16,
|
||
"type": "logs",
|
||
"title": "Recent backup runs",
|
||
"description": "Agents backed up, skipped and failed by hive-archive-backup, newest first.",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"gridPos": {
|
||
"x": 12,
|
||
"y": 49,
|
||
"w": 12,
|
||
"h": 9
|
||
},
|
||
"targets": [
|
||
{
|
||
"refId": "A",
|
||
"datasource": {
|
||
"type": "victoriametrics-logs-datasource",
|
||
"uid": "@logsDatasourceUid@"
|
||
},
|
||
"expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\"",
|
||
"queryType": "instant"
|
||
}
|
||
],
|
||
"options": {
|
||
"showTime": true,
|
||
"wrapLogMessage": true,
|
||
"prettifyLogMessage": false,
|
||
"enableLogDetails": true,
|
||
"sortOrder": "Descending",
|
||
"dedupStrategy": "none"
|
||
}
|
||
}
|
||
]
|
||
}
|