agent: export swarm-queue connected gauge and skipped-publish counter, with dashboard panels
This commit is contained in:
atlas
mara
parent
04f61c0e14
commit
e9025823d9
2 changed files with 381 additions and 24 deletions
|
|
@ -3698,6 +3698,182 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-83": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 83,
|
||||
"title": "Swarm queue connection by agent",
|
||||
"description": "Whether each agent is connected to the swarm queue. Grey: no data, agent on an older build.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "max by (agent) (last_over_time({__name__=\"hyperhive.agent.swarm_queue.connected\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) or (max by (agent) (last_over_time({__name__=\"hyperhive.container.cpu.percent\", hive=~\"$hive\", agent=~\"$agent\"}[5m])) * 0 - 1)",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "state-timeline",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"alignValue": "left",
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"mergeValues": true,
|
||||
"rowHeight": 0.9,
|
||||
"showValue": "auto",
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"fillOpacity": 80,
|
||||
"lineWidth": 0
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"1": {
|
||||
"text": "connected",
|
||||
"color": "green",
|
||||
"index": 0
|
||||
},
|
||||
"0": {
|
||||
"text": "disconnected",
|
||||
"color": "red",
|
||||
"index": 1
|
||||
},
|
||||
"-1": {
|
||||
"text": "no data",
|
||||
"color": "text",
|
||||
"index": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"value": null,
|
||||
"color": "text"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-84": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"id": 84,
|
||||
"title": "Skipped queue publishes per hour by agent",
|
||||
"description": "Publishes an agent dropped because it was not connected to the swarm queue, per hour.",
|
||||
"links": [],
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"query": {
|
||||
"kind": "DataQuery",
|
||||
"group": "prometheus",
|
||||
"version": "v0",
|
||||
"datasource": {
|
||||
"name": "@datasourceUid@"
|
||||
},
|
||||
"spec": {
|
||||
"expr": "sum by (agent) (rate({__name__=\"hyperhive.agent.swarm_queue.publish_skipped\", hive=~\"$hive\", agent=~\"$agent\"}[$__rate_interval])) * 3600",
|
||||
"legendFormat": "{{agent}}"
|
||||
}
|
||||
},
|
||||
"refId": "A",
|
||||
"hidden": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"transformations": [],
|
||||
"queryOptions": {}
|
||||
}
|
||||
},
|
||||
"vizConfig": {
|
||||
"kind": "VizConfig",
|
||||
"group": "timeseries",
|
||||
"version": "",
|
||||
"spec": {
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisSoftMin": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"lineWidth": 1,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"layout": {
|
||||
|
|
@ -4270,6 +4446,32 @@
|
|||
"name": "panel-68"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 39,
|
||||
"width": 24,
|
||||
"height": 7,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-83"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"x": 0,
|
||||
"y": 46,
|
||||
"width": 24,
|
||||
"height": 8,
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-84"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -505,6 +505,32 @@ let
|
|||
]
|
||||
&& (lib.head xf).spec.options.byField == "agent";
|
||||
}
|
||||
{
|
||||
# The agents board's health tab reads the swarm-queue connection gauge and
|
||||
# the skipped-publish counter the harness records. A renamed instrument is a
|
||||
# pair of empty panels, so each must still be spelled in the harness; and the
|
||||
# connection panel only tells an older build (no series) from a detached
|
||||
# agent (0) while it falls back to the container metrics for the agent list.
|
||||
name = "the agents dashboard's health tab reads the swarm-queue metrics the harness records";
|
||||
ok =
|
||||
let
|
||||
board = builtins.fromJSON (builtins.readFile ../host-modules/swarm-grafana/dashboards/agents.json);
|
||||
tab = lib.findFirst (t: t.spec.title == "health") null board.spec.layout.spec.tabs;
|
||||
names = map (i: i.spec.element.name) tab.spec.layout.spec.items;
|
||||
exprs = lib.concatMap (
|
||||
n: map (q: q.spec.query.spec.expr) board.spec.elements.${n}.spec.data.spec.queries
|
||||
) names;
|
||||
recorded = builtins.readFile ../../hive-agent/src/otel_turn_metrics.rs;
|
||||
connected = lib.findFirst (e: lib.hasInfix "swarm_queue.connected" e) "" exprs;
|
||||
in
|
||||
tab != null
|
||||
&& lib.all (m: lib.hasInfix "\"${m}\"" recorded) [
|
||||
"hyperhive.agent.swarm_queue.connected"
|
||||
"hyperhive.agent.swarm_queue.publish_skipped"
|
||||
]
|
||||
&& lib.any (e: lib.hasInfix "swarm_queue.publish_skipped" e) exprs
|
||||
&& lib.hasInfix "hyperhive.container.cpu.percent" connected;
|
||||
}
|
||||
{
|
||||
# The agents board's turns tab reads the turn, tool, bash command, reminder
|
||||
# and session metrics the harness records, grouped by the labels it sets on them. A renamed instrument or
|
||||
|
|
|
|||
Loading…
Reference in a new issue