{
  "title": "hyperhive · backups (restic)",
  "uid": "hyperhive-archive",
  "description": "Each agent's restic backup to the swarm archive, and the archive's prune.",
  "editable": true,
  "refresh": "1m",
  "schemaVersion": 39,
  "tags": ["hyperhive", "swarm", "archive", "backup", "restic"],
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timezone": "utc",
  "version": 1,
  "templating": {
    "list": [
      {
        "name": "hive",
        "label": "Hive",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "@datasourceUid@"
        },
        "query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\"}, hive)",
        "refresh": 2,
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "sort": 1
      },
      {
        "name": "agent",
        "label": "Agent",
        "type": "query",
        "datasource": {
          "type": "prometheus",
          "uid": "@datasourceUid@"
        },
        "query": "label_values({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\"}, agent)",
        "refresh": 2,
        "includeAll": true,
        "allValue": ".*",
        "multi": true,
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "sort": 1
      }
    ]
  },
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "Backups",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 0,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Agents past 2× the interval",
      "description": "Agents whose last successful backup is over 2 hours old, twice the hourly schedule.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 1,
        "w": 6,
        "h": 4
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "count(time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) > 7200) or vector(0)",
          "instant": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "red"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Agents whose last run failed",
      "description": "Agents whose latest backup run ended in failure. Source: last_run_success gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 6,
        "y": 1,
        "w": 6,
        "h": 4
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d])) == 0) or vector(0)",
          "instant": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "red"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 4,
      "type": "stat",
      "title": "Agents backed up",
      "description": "Agents that reported a backup run in the last 7 days. Source: last_run_success gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 12,
        "y": 1,
        "w": 6,
        "h": 4
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "count(max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))) or vector(0)",
          "instant": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "fixed",
            "fixedColor": "green"
          },
          "mappings": []
        },
        "overrides": []
      },
      "options": {
        "graphMode": "none",
        "colorMode": "value",
        "textMode": "value",
        "justifyMode": "auto",
        "reduceOptions": {
          "calcs": ["lastNotNull"],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 5,
      "type": "table",
      "title": "Last backup per agent",
      "description": "Per agent: time since last success (red over 2 hours), last result, duration, data added.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 5,
        "w": 24,
        "h": 9
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_success_timestamp_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.last_run_success\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "C",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "D",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        }
      ],
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "agent",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true
            },
            "renameByName": {
              "agent": "Agent",
              "Value #A": "Since last success",
              "Value #B": "Last result",
              "Value #C": "Duration",
              "Value #D": "Data added"
            }
          }
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Since last success"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              },
              {
                "id": "custom.cellOptions",
                "value": {
                  "type": "color-text"
                }
              },
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": null
                    },
                    {
                      "color": "red",
                      "value": 7200
                    }
                  ]
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Last result"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "type": "value",
                    "options": {
                      "0": {
                        "text": "failed",
                        "color": "red",
                        "index": 0
                      },
                      "1": {
                        "text": "ok",
                        "color": "green",
                        "index": 1
                      }
                    }
                  }
                ]
              },
              {
                "id": "custom.cellOptions",
                "value": {
                  "type": "color-text"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Duration"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Data added"
            },
            "properties": [
              {
                "id": "unit",
                "value": "bytes"
              }
            ]
          }
        ]
      },
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "Since last success",
            "desc": true
          }
        ]
      }
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Data added per run",
      "description": "Bytes each backup run added to the repository, per agent. Source: data_added_bytes gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 14,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.data_added_bytes\", hive=~\"$hive\", agent=~\"$agent\"})",
          "legendFormat": "{{agent}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes",
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 10,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Backup duration",
      "description": "Wall time of each agent's backup run, snapshot included. Source: duration_seconds gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 12,
        "y": 14,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) ({__name__=\"hyperhive.archive.backup.duration_seconds\", hive=~\"$hive\", agent=~\"$agent\"})",
          "legendFormat": "{{agent}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 10,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 8,
      "type": "row",
      "title": "Repositories",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 22,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 9,
      "type": "table",
      "title": "Repository and prune per agent",
      "description": "Per agent: snapshots, repository size, newest snapshot age, last prune age and result.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 23,
        "w": 24,
        "h": 9
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "C",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.repo.latest_snapshot_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "D",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "time() - max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_timestamp_seconds\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        },
        {
          "refId": "E",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) (last_over_time({__name__=\"hyperhive.archive.prune.last_run_success\", agent=~\"$agent\"}[7d]))",
          "instant": true,
          "format": "table"
        }
      ],
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "agent",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true
            },
            "renameByName": {
              "agent": "Agent",
              "Value #A": "Snapshots",
              "Value #B": "Size",
              "Value #C": "Newest snapshot age",
              "Value #D": "Since last prune",
              "Value #E": "Prune result"
            }
          }
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": "auto"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Size"
            },
            "properties": [
              {
                "id": "unit",
                "value": "bytes"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Newest snapshot age"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Since last prune"
            },
            "properties": [
              {
                "id": "unit",
                "value": "s"
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Prune result"
            },
            "properties": [
              {
                "id": "mappings",
                "value": [
                  {
                    "type": "value",
                    "options": {
                      "0": {
                        "text": "failed",
                        "color": "red",
                        "index": 0
                      },
                      "1": {
                        "text": "ok",
                        "color": "green",
                        "index": 1
                      }
                    }
                  }
                ]
              },
              {
                "id": "custom.cellOptions",
                "value": {
                  "type": "color-text"
                }
              }
            ]
          }
        ]
      },
      "options": {
        "showHeader": true,
        "sortBy": [
          {
            "displayName": "Agent",
            "desc": true
          }
        ]
      }
    },
    {
      "id": 10,
      "type": "timeseries",
      "title": "Snapshots per repository",
      "description": "Snapshots each agent's repository holds after the prune. Source: repo.snapshots gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 32,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.snapshots\", agent=~\"$agent\"})",
          "legendFormat": "{{agent}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 10,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 11,
      "type": "timeseries",
      "title": "Repository size",
      "description": "Size of the data in each agent's repository. Source: repo.size_bytes gauge.",
      "datasource": {
        "type": "prometheus",
        "uid": "@datasourceUid@"
      },
      "gridPos": {
        "x": 12,
        "y": 32,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "prometheus",
            "uid": "@datasourceUid@"
          },
          "expr": "max by (agent) ({__name__=\"hyperhive.archive.repo.size_bytes\", agent=~\"$agent\"})",
          "legendFormat": "{{agent}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "bytes",
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "lineWidth": 1,
            "fillOpacity": 10,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 12,
      "type": "row",
      "title": "Failures",
      "collapsed": false,
      "gridPos": {
        "x": 0,
        "y": 40,
        "w": 24,
        "h": 1
      },
      "panels": []
    },
    {
      "id": 13,
      "type": "timeseries",
      "title": "Failures over time",
      "description": "Backup and prune failure lines per interval.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 41,
        "w": 12,
        "h": 8
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\" | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "backup failed"
        },
        {
          "refId": "B",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\" | stats count_uniq(__CURSOR) as n",
          "queryType": "statsRange",
          "legendFormat": "prune failed"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "short",
          "decimals": 0,
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "lineWidth": 1,
            "fillOpacity": 60,
            "showPoints": "never",
            "spanNulls": false,
            "axisSoftMin": 0
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true,
          "calcs": []
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      }
    },
    {
      "id": 14,
      "type": "logs",
      "title": "Recent backup failures",
      "description": "Agents hive-archive-backup could not back up, newest first.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "x": 12,
        "y": 41,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\" _msg:\"archive backup failed\"",
          "queryType": "instant"
        }
      ],
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "prettifyLogMessage": false,
        "enableLogDetails": true,
        "sortOrder": "Descending",
        "dedupStrategy": "none"
      }
    },
    {
      "id": 15,
      "type": "logs",
      "title": "Recent prune failures",
      "description": "Repositories swarm-archive-prune could not prune, newest first.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "x": 0,
        "y": 49,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:=\"swarm-archive-prune.service\" _msg:\"archive prune failed\"",
          "queryType": "instant"
        }
      ],
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "prettifyLogMessage": false,
        "enableLogDetails": true,
        "sortOrder": "Descending",
        "dedupStrategy": "none"
      }
    },
    {
      "id": 16,
      "type": "logs",
      "title": "Recent backup runs",
      "description": "Agents backed up, skipped and failed by hive-archive-backup, newest first.",
      "datasource": {
        "type": "victoriametrics-logs-datasource",
        "uid": "@logsDatasourceUid@"
      },
      "gridPos": {
        "x": 12,
        "y": 49,
        "w": 12,
        "h": 9
      },
      "targets": [
        {
          "refId": "A",
          "datasource": {
            "type": "victoriametrics-logs-datasource",
            "uid": "@logsDatasourceUid@"
          },
          "expr": "_SYSTEMD_UNIT:=\"hive-archive-backup.service\"",
          "queryType": "instant"
        }
      ],
      "options": {
        "showTime": true,
        "wrapLogMessage": true,
        "prettifyLogMessage": false,
        "enableLogDetails": true,
        "sortOrder": "Descending",
        "dedupStrategy": "none"
      }
    }
  ]
}