add a grafana dashboard for the swarm binary cache
This commit is contained in:
atlas
mara
parent
c39a836bc5
commit
b2694b0a92
3 changed files with 813 additions and 0 deletions
|
|
@ -201,6 +201,7 @@ Grafana start — see [`credentials.md`](credentials.md).
|
||||||
kept.
|
kept.
|
||||||
|
|
||||||
- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first.
|
- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first.
|
||||||
|
- `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused. It reads the closure-pullable gauge and the `swarm-builder` and `hive-c0re` logs. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request, hit or storage panel. Filter by hive and agent.
|
||||||
- `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue.
|
- `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue.
|
||||||
- `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents.
|
- `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents.
|
||||||
- `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, and the busiest agents and destinations. **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent.
|
- `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, and the busiest agents and destinations. **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent.
|
||||||
|
|
|
||||||
|
|
@ -269,6 +269,7 @@ let
|
||||||
})
|
})
|
||||||
[
|
[
|
||||||
"agents.json"
|
"agents.json"
|
||||||
|
"attic.json"
|
||||||
"authelia.json"
|
"authelia.json"
|
||||||
"bao.json"
|
"bao.json"
|
||||||
"builds.json"
|
"builds.json"
|
||||||
|
|
|
||||||
811
nix/host-modules/swarm-grafana/dashboards/attic.json
Normal file
811
nix/host-modules/swarm-grafana/dashboards/attic.json
Normal file
|
|
@ -0,0 +1,811 @@
|
||||||
|
{
|
||||||
|
"title": "hyperhive · binary cache",
|
||||||
|
"uid": "hyperhive-attic",
|
||||||
|
"description": "The swarm's binary cache: what the builder pushes, what each hive can pull, and the refusals.",
|
||||||
|
"editable": true,
|
||||||
|
"refresh": "1m",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["hyperhive", "swarm", "attic", "cache"],
|
||||||
|
"time": {
|
||||||
|
"from": "now-24h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timezone": "utc",
|
||||||
|
"version": 1,
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "hive",
|
||||||
|
"label": "Hive",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"query": "label_values({__name__=\"hyperhive.agent.closure_pullable\"}, hive)",
|
||||||
|
"refresh": 2,
|
||||||
|
"includeAll": true,
|
||||||
|
"allValue": ".*",
|
||||||
|
"multi": true,
|
||||||
|
"current": {
|
||||||
|
"text": "All",
|
||||||
|
"value": "$__all"
|
||||||
|
},
|
||||||
|
"sort": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "agent",
|
||||||
|
"label": "Agent",
|
||||||
|
"type": "query",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"query": "label_values({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\"}, agent)",
|
||||||
|
"refresh": 2,
|
||||||
|
"includeAll": true,
|
||||||
|
"allValue": ".*",
|
||||||
|
"multi": false,
|
||||||
|
"current": {
|
||||||
|
"text": "All",
|
||||||
|
"value": "$__all"
|
||||||
|
},
|
||||||
|
"sort": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Pullable closures",
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Agents pullable",
|
||||||
|
"description": "Agents whose latest closure this hive can pull and verify. Source: closure_pullable gauge.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 4,
|
||||||
|
"w": 6,
|
||||||
|
"x": 0,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "sum(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))) or vector(0)",
|
||||||
|
"instant": true,
|
||||||
|
"legendFormat": "pullable"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "green"
|
||||||
|
},
|
||||||
|
"mappings": []
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"graphMode": "none",
|
||||||
|
"colorMode": "value",
|
||||||
|
"textMode": "value",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Agents not pullable",
|
||||||
|
"description": "Agents whose latest closure this hive cannot pull. Source: closure_pullable gauge.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 4,
|
||||||
|
"w": 6,
|
||||||
|
"x": 6,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "count(max by (hive, agent) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m])) == 0) or vector(0)",
|
||||||
|
"instant": true,
|
||||||
|
"legendFormat": "not pullable"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "red"
|
||||||
|
},
|
||||||
|
"mappings": []
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"graphMode": "none",
|
||||||
|
"colorMode": "value",
|
||||||
|
"textMode": "value",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Closures pushed (range)",
|
||||||
|
"description": "Closures the builder pushed to the cache in the range. Source: swarm-builder logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 4,
|
||||||
|
"w": 6,
|
||||||
|
"x": 12,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "stats",
|
||||||
|
"legendFormat": "pushed"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "green"
|
||||||
|
},
|
||||||
|
"mappings": []
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"graphMode": "none",
|
||||||
|
"colorMode": "value",
|
||||||
|
"textMode": "value",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Push failures (range)",
|
||||||
|
"description": "Closures the builder failed to push in the range. Source: swarm-builder logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 4,
|
||||||
|
"w": 6,
|
||||||
|
"x": 18,
|
||||||
|
"y": 1
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "stats",
|
||||||
|
"legendFormat": "failed"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "red"
|
||||||
|
},
|
||||||
|
"mappings": []
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"graphMode": "none",
|
||||||
|
"colorMode": "value",
|
||||||
|
"textMode": "value",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "table",
|
||||||
|
"title": "Pull verdict per agent",
|
||||||
|
"description": "Per agent: whether the latest closure is pullable, and why not. Source: closure_pullable gauge.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 5
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "max by (agent, reason) (last_over_time({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}[3m]))",
|
||||||
|
"instant": true,
|
||||||
|
"format": "table"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"id": "organize",
|
||||||
|
"options": {
|
||||||
|
"excludeByName": {
|
||||||
|
"Time": true
|
||||||
|
},
|
||||||
|
"renameByName": {
|
||||||
|
"agent": "Agent",
|
||||||
|
"reason": "Reason",
|
||||||
|
"Value": "Pullable"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"custom": {
|
||||||
|
"align": "auto"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": [
|
||||||
|
{
|
||||||
|
"matcher": {
|
||||||
|
"id": "byName",
|
||||||
|
"options": "Pullable"
|
||||||
|
},
|
||||||
|
"properties": [
|
||||||
|
{
|
||||||
|
"id": "mappings",
|
||||||
|
"value": [
|
||||||
|
{
|
||||||
|
"type": "value",
|
||||||
|
"options": {
|
||||||
|
"0": {
|
||||||
|
"text": "no",
|
||||||
|
"color": "red",
|
||||||
|
"index": 0
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"text": "yes",
|
||||||
|
"color": "green",
|
||||||
|
"index": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "custom.cellOptions",
|
||||||
|
"value": {
|
||||||
|
"type": "color-text"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"showHeader": true,
|
||||||
|
"sortBy": [
|
||||||
|
{
|
||||||
|
"displayName": "Pullable",
|
||||||
|
"desc": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Pullable agents over time",
|
||||||
|
"description": "Agents with a pullable closure against agents without. Source: closure_pullable gauge.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 5
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "sum(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"})) or vector(0)",
|
||||||
|
"legendFormat": "pullable"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "@datasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "count(max by (hive, agent) ({__name__=\"hyperhive.agent.closure_pullable\", hive=~\"$hive\", agent=~\"$agent\"}) == 0) or vector(0)",
|
||||||
|
"legendFormat": "not pullable"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "line",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Builder pushes",
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 14
|
||||||
|
},
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Pushes over time",
|
||||||
|
"description": "Closures pushed and not pushed to the cache, per interval. Source: swarm-builder logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 15
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure pushed to the cache\" | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "pushed"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\" | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "not pushed"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Agent meta builds",
|
||||||
|
"description": "Agent builds from the meta flake by outcome, per interval. Source: swarm-builder logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 15
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"agent meta build built\" | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "built"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" (\"agent meta build failed\" OR \"agent meta build not run\" OR \"meta build refused\") | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "failed or refused"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Recent push failures",
|
||||||
|
"description": "The builder's push-failure lines with the reason, newest first. Source: swarm-builder logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 9,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 23
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"swarm-builder.service\" F_AGENT:~\"$agent\" \"closure not pushed to the cache\"",
|
||||||
|
"queryType": "instant"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"options": {
|
||||||
|
"showTime": true,
|
||||||
|
"wrapLogMessage": true,
|
||||||
|
"prettifyLogMessage": false,
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"sortOrder": "Descending",
|
||||||
|
"dedupStrategy": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 20,
|
||||||
|
"type": "row",
|
||||||
|
"title": "Hive pulls",
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 32
|
||||||
|
},
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 21,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Closures substituted (range)",
|
||||||
|
"description": "Closures hive-c0re pulled from the cache for a prebuild or create. Source: hive-c0re logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 4,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 33
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "stats",
|
||||||
|
"legendFormat": "substituted"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "fixed",
|
||||||
|
"fixedColor": "green"
|
||||||
|
},
|
||||||
|
"mappings": []
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"graphMode": "none",
|
||||||
|
"colorMode": "value",
|
||||||
|
"textMode": "value",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": ["lastNotNull"],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 23,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Substitutes and refusals",
|
||||||
|
"description": "Closures pulled and deploy steps refused, per interval. Source: hive-c0re logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 37
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" (\"prebuild: substituted\" OR \"create: substituted\") | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "substituted"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_NAME:~\"$agent\" \"nothing was changed\" (\"the swarm has recorded no build\" OR \"closure cannot be pulled\") | stats count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "refused"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 24,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Probe verdict changes by reason",
|
||||||
|
"description": "Times the pull probe's verdict for an agent changed, by reason. Source: hive-c0re logs.",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 45
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"datasource": {
|
||||||
|
"type": "victoriametrics-logs-datasource",
|
||||||
|
"uid": "@logsDatasourceUid@"
|
||||||
|
},
|
||||||
|
"expr": "_SYSTEMD_UNIT:\"hive-c0re.service\" F_AGENT:~\"$agent\" \"attic: can this hive pull\" F_REASON:* | stats by (F_REASON) count_uniq(__CURSOR) as n",
|
||||||
|
"queryType": "statsRange",
|
||||||
|
"legendFormat": "{{F_REASON}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"drawStyle": "bars",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"fillOpacity": 60,
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"axisSoftMin": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true,
|
||||||
|
"calcs": []
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "desc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue