Compare commits
5 changed files with 8 additions and 676 deletions
|
|
@ -195,9 +195,9 @@ let
|
||||||
|
|
||||||
# Metrics get their own listener rather than a flag on the one above, and
|
# Metrics get their own listener rather than a flag on the one above, and
|
||||||
# that follows from what a scraper can express: `swarm.otel.scrapeTargets`
|
# that follows from what a scraper can express: `swarm.otel.scrapeTargets`
|
||||||
# carries no scheme and no credential, while the API listener is TLS and —
|
# is `host:port` with no scheme and no credential, while the API listener is
|
||||||
# once a client CA is set — demands a client certificate. The collector
|
# TLS and — once a client CA is set — demands a client certificate. The
|
||||||
# cannot reach it at all.
|
# collector cannot reach it at all.
|
||||||
#
|
#
|
||||||
# `metrics_only` narrows this one to the metrics path (every other path 404s)
|
# `metrics_only` narrows this one to the metrics path (every other path 404s)
|
||||||
# and the unauthenticated access is confined to loopback. **Deliberately
|
# and the unauthenticated access is confined to loopback. **Deliberately
|
||||||
|
|
@ -576,11 +576,7 @@ in
|
||||||
];
|
];
|
||||||
|
|
||||||
services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
|
services.hyperhive.swarm.otel.scrapeTargets = lib.mkIf scrapeHere {
|
||||||
# Path and query, not just `host:port`: openbao serves no `/metrics`
|
bao = "127.0.0.1:${toString baoDeploy.metricsPort}";
|
||||||
# at all, and `/v1/sys/metrics` answers JSON unless the format is
|
|
||||||
# asked for. A scrape of the default path 404s, which reads as a
|
|
||||||
# dead exporter rather than a wrong address.
|
|
||||||
bao = "127.0.0.1:${toString baoDeploy.metricsPort}/v1/sys/metrics?format=prometheus";
|
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,6 @@ let
|
||||||
[
|
[
|
||||||
"agents.json"
|
"agents.json"
|
||||||
"authelia.json"
|
"authelia.json"
|
||||||
"bao.json"
|
|
||||||
"claude-usage.json"
|
"claude-usage.json"
|
||||||
"forge.json"
|
"forge.json"
|
||||||
"logstore.json"
|
"logstore.json"
|
||||||
|
|
|
||||||
|
|
@ -1,569 +0,0 @@
|
||||||
{
|
|
||||||
"title": "hyperhive · openbao",
|
|
||||||
"uid": "hyperhive-bao",
|
|
||||||
"description": "The swarm's secret store. Two things to know before editing: the metric prefix is vault_, not openbao_ — the fork kept upstream's names, and a panel built on the name you would guess renders perfectly and shows nothing. And a SEALED store emits almost nothing: only the top row works before an operator has run `bao operator init`, everything below it is empty until then and that is the expected reading, not a broken query. Targets are labelled service.name, NOT job; there is no job label in this store. Counters are counts over the dashboard range rather than per-second rates, because a store this quiet rates to exactly zero.",
|
|
||||||
"editable": true,
|
|
||||||
"refresh": "1m",
|
|
||||||
"schemaVersion": 39,
|
|
||||||
"tags": [
|
|
||||||
"hyperhive",
|
|
||||||
"swarm",
|
|
||||||
"bao",
|
|
||||||
"secrets"
|
|
||||||
],
|
|
||||||
"time": {
|
|
||||||
"from": "now-24h",
|
|
||||||
"to": "now"
|
|
||||||
},
|
|
||||||
"timezone": "utc",
|
|
||||||
"version": 1,
|
|
||||||
"templating": {
|
|
||||||
"list": []
|
|
||||||
},
|
|
||||||
"panels": [
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Seal state",
|
|
||||||
"description": "SEALED means the store is up but refusing every read — the state it boots into, and the state it returns to on restart unless auto-unseal is configured. The max() is load-bearing: once unsealed there are two vault_core_unsealed series and the cluster=\"\" one stays 0 forever, so a bare selector reads SEALED on a healthy store.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 6,
|
|
||||||
"x": 0,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "max(vault_core_unsealed{\"service.name\"=\"bao\"})",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "seal"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "thresholds"
|
|
||||||
},
|
|
||||||
"mappings": [
|
|
||||||
{
|
|
||||||
"type": "value",
|
|
||||||
"options": {
|
|
||||||
"0": {
|
|
||||||
"text": "SEALED",
|
|
||||||
"color": "red",
|
|
||||||
"index": 0
|
|
||||||
},
|
|
||||||
"1": {
|
|
||||||
"text": "unsealed",
|
|
||||||
"color": "green",
|
|
||||||
"index": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"thresholds": {
|
|
||||||
"mode": "absolute",
|
|
||||||
"steps": [
|
|
||||||
{
|
|
||||||
"color": "red",
|
|
||||||
"value": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"color": "green",
|
|
||||||
"value": 1
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "background",
|
|
||||||
"textMode": "value",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 2,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Uptime",
|
|
||||||
"description": "Time since the openbao process started. Read it beside the seal state: a restart re-seals the store, so a short uptime and a SEALED panel are one event rather than two problems.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 6,
|
|
||||||
"x": 6,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "time() - process_start_time_seconds{\"service.name\"=\"bao\"}",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "uptime"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "s",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "text"
|
|
||||||
},
|
|
||||||
"mappings": []
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "value",
|
|
||||||
"textMode": "value",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 3,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Goroutines",
|
|
||||||
"description": "A Go runtime metric, so it is present even while sealed — which makes it the panel that distinguishes a store with nothing to say from a scrape that is not arriving at all. No data here means the collector is not reaching the metrics listener.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 6,
|
|
||||||
"x": 12,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "go_goroutines{\"service.name\"=\"bao\"}",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "goroutines"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "text"
|
|
||||||
},
|
|
||||||
"mappings": []
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "value",
|
|
||||||
"textMode": "value",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 4,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Requests (range)",
|
|
||||||
"description": "vault_core_handle_request_count over the dashboard range — every API call the store served, unseal attempts included. The summary's _count series, not the summary itself. Zero on a sealed store, and zero is the honest answer there rather than an absent panel.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 6,
|
|
||||||
"x": 18,
|
|
||||||
"y": 0
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "sum(increase(vault_core_handle_request_count{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "requests"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "text"
|
|
||||||
},
|
|
||||||
"mappings": []
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "value",
|
|
||||||
"textMode": "value",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 10,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Seal operations (range)",
|
|
||||||
"description": "Encrypt and decrypt calls against the seal wrapper — on this swarm a TPM through PKCS#11, so each one is a round trip to hardware. These are counters with no _total suffix. A climbing decrypt count with a flat request count means something is re-reading the barrier rather than serving clients.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 12,
|
|
||||||
"x": 0,
|
|
||||||
"y": 4
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "sum(increase(vault_seal_encrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "encrypt"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "sum(increase(vault_seal_decrypt{\"service.name\"=\"bao\"}[$__range])) or vector(0)",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "decrypt"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"mappings": []
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "value",
|
|
||||||
"textMode": "value_and_name",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 11,
|
|
||||||
"type": "stat",
|
|
||||||
"title": "Raft peers",
|
|
||||||
"description": "Voting members of the store's raft cluster. This swarm runs a single node, so 1 is the expected value and anything else means a peer was added — or that this node no longer sees itself as part of the cluster it stores data for.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 4,
|
|
||||||
"w": 12,
|
|
||||||
"x": 12,
|
|
||||||
"y": 4
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "vault_raft_peers{\"service.name\"=\"bao\"}",
|
|
||||||
"instant": true,
|
|
||||||
"legendFormat": "peers"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "fixed",
|
|
||||||
"fixedColor": "text"
|
|
||||||
},
|
|
||||||
"mappings": []
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"graphMode": "none",
|
|
||||||
"colorMode": "value",
|
|
||||||
"textMode": "value",
|
|
||||||
"justifyMode": "auto",
|
|
||||||
"reduceOptions": {
|
|
||||||
"calcs": [
|
|
||||||
"lastNotNull"
|
|
||||||
],
|
|
||||||
"fields": "",
|
|
||||||
"values": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 20,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Request latency",
|
|
||||||
"description": "Quantiles openbao computes itself — a summary, so the store picked the buckets and a histogram_quantile() is neither needed nor possible here. Units are MILLISECONDS: vault's timers report ms, and reading them as seconds understates the store by three orders of magnitude.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 8,
|
|
||||||
"w": 12,
|
|
||||||
"x": 0,
|
|
||||||
"y": 8
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.99\"}",
|
|
||||||
"legendFormat": "p99"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "vault_core_handle_request{\"service.name\"=\"bao\",quantile=\"0.5\"}",
|
|
||||||
"legendFormat": "p50"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "ms",
|
|
||||||
"decimals": 1,
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "line",
|
|
||||||
"lineWidth": 1,
|
|
||||||
"fillOpacity": 0,
|
|
||||||
"showPoints": "never",
|
|
||||||
"spanNulls": false,
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": []
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 21,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Leases",
|
|
||||||
"description": "Outstanding leases, and the irrevocable ones beside them. The second line should sit at zero: an irrevocable lease is one the store gave up trying to revoke, so a credential somewhere outlives what issued it and no amount of waiting will clean it up.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 8,
|
|
||||||
"w": 12,
|
|
||||||
"x": 12,
|
|
||||||
"y": 8
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "vault_expire_num_leases{\"service.name\"=\"bao\"}",
|
|
||||||
"legendFormat": "leases"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"refId": "B",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "vault_expire_num_irrevocable_leases{\"service.name\"=\"bao\"}",
|
|
||||||
"legendFormat": "irrevocable"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "short",
|
|
||||||
"decimals": 0,
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "line",
|
|
||||||
"lineWidth": 1,
|
|
||||||
"fillOpacity": 10,
|
|
||||||
"showPoints": "never",
|
|
||||||
"spanNulls": false,
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": []
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 30,
|
|
||||||
"type": "timeseries",
|
|
||||||
"title": "Heap in use",
|
|
||||||
"description": "go_memstats_heap_inuse_bytes. Present while sealed, so it is a live line even on a store that is serving nothing — a sawtooth is normal Go behaviour, a monotonic climb on an idle store is not.",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"gridPos": {
|
|
||||||
"h": 6,
|
|
||||||
"w": 24,
|
|
||||||
"x": 0,
|
|
||||||
"y": 16
|
|
||||||
},
|
|
||||||
"targets": [
|
|
||||||
{
|
|
||||||
"refId": "A",
|
|
||||||
"datasource": {
|
|
||||||
"type": "prometheus",
|
|
||||||
"uid": "@datasourceUid@"
|
|
||||||
},
|
|
||||||
"expr": "go_memstats_heap_inuse_bytes{\"service.name\"=\"bao\"}",
|
|
||||||
"legendFormat": "heap"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"fieldConfig": {
|
|
||||||
"defaults": {
|
|
||||||
"unit": "bytes",
|
|
||||||
"decimals": 1,
|
|
||||||
"color": {
|
|
||||||
"mode": "palette-classic"
|
|
||||||
},
|
|
||||||
"custom": {
|
|
||||||
"drawStyle": "line",
|
|
||||||
"lineWidth": 1,
|
|
||||||
"fillOpacity": 10,
|
|
||||||
"showPoints": "never",
|
|
||||||
"spanNulls": false,
|
|
||||||
"axisSoftMin": 0
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"overrides": []
|
|
||||||
},
|
|
||||||
"options": {
|
|
||||||
"legend": {
|
|
||||||
"displayMode": "list",
|
|
||||||
"placement": "bottom",
|
|
||||||
"showLegend": true,
|
|
||||||
"calcs": []
|
|
||||||
},
|
|
||||||
"tooltip": {
|
|
||||||
"mode": "multi",
|
|
||||||
"sort": "desc"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -91,46 +91,6 @@ let
|
||||||
# error rather than a warning nobody reads.
|
# error rather than a warning nobody reads.
|
||||||
parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
|
parsePublished = url: builtins.match "https://([^/]+)(/.*)" url;
|
||||||
|
|
||||||
# A loopback target may carry a path and a query as well as `host:port`,
|
|
||||||
# because not every exporter serves `/metrics`: openbao 404s there and
|
|
||||||
# answers on `/v1/sys/metrics?format=prometheus`. Split for the same reason
|
|
||||||
# the published one is — one declaration, three prometheus fields.
|
|
||||||
#
|
|
||||||
# ⚠️ The query may not ride along in `metrics_path`: prometheus percent-
|
|
||||||
# encodes the `?`, so the request goes to a path that does not exist. It has
|
|
||||||
# to become `params`, whose values are lists.
|
|
||||||
#
|
|
||||||
# `null` when the shape is wrong, which the assertion below reports by name.
|
|
||||||
parseScrape = t: builtins.match "([^/?]+)(/[^?]*)?(\\?(.+))?" t;
|
|
||||||
|
|
||||||
queryParams =
|
|
||||||
query:
|
|
||||||
lib.listToAttrs (
|
|
||||||
map (
|
|
||||||
kv:
|
|
||||||
let
|
|
||||||
parts = lib.splitString "=" kv;
|
|
||||||
in
|
|
||||||
lib.nameValuePair (lib.head parts) [ (lib.concatStringsSep "=" (lib.tail parts)) ]
|
|
||||||
) (lib.splitString "&" query)
|
|
||||||
);
|
|
||||||
|
|
||||||
# Both optional fields are omitted rather than defaulted, so a plain
|
|
||||||
# `host:port` renders the config it rendered before this grammar existed.
|
|
||||||
loopbackScrapeConfig =
|
|
||||||
job: target:
|
|
||||||
let
|
|
||||||
parts = parseScrape target;
|
|
||||||
path = lib.elemAt parts 1;
|
|
||||||
query = lib.elemAt parts 3;
|
|
||||||
in
|
|
||||||
{
|
|
||||||
job_name = job;
|
|
||||||
static_configs = [ { targets = [ (lib.elemAt parts 0) ]; } ];
|
|
||||||
}
|
|
||||||
// lib.optionalAttrs (path != null) { metrics_path = path; }
|
|
||||||
// lib.optionalAttrs (query != null) { params = queryParams query; };
|
|
||||||
|
|
||||||
# The client secret takes three names, and the reason is `DynamicUser`.
|
# The client secret takes three names, and the reason is `DynamicUser`.
|
||||||
#
|
#
|
||||||
# Upstream's collector unit runs with `DynamicUser = true`, and the
|
# Upstream's collector unit runs with `DynamicUser = true`, and the
|
||||||
|
|
@ -372,12 +332,6 @@ in
|
||||||
Prometheus exposition endpoints this collector scrapes, as
|
Prometheus exposition endpoints this collector scrapes, as
|
||||||
`<job name> = "<host>:<port>"`.
|
`<job name> = "<host>:<port>"`.
|
||||||
|
|
||||||
A path and query may follow the port —
|
|
||||||
`"127.0.0.1:8202/v1/sys/metrics?format=prometheus"` — for an
|
|
||||||
exporter that does not serve `/metrics`. Both are optional and
|
|
||||||
omitted when absent, so a bare `host:port` is scraped exactly as
|
|
||||||
before.
|
|
||||||
|
|
||||||
**A service declares its own entry, from its own module, under its
|
**A service declares its own entry, from its own module, under its
|
||||||
own `enable`.** Do not assemble the list here: an entry then exists
|
own `enable`.** Do not assemble the list here: an entry then exists
|
||||||
only where the service that named it runs, so a target is never
|
only where the service that named it runs, so a target is never
|
||||||
|
|
@ -757,23 +711,6 @@ in
|
||||||
token, and plain http would put it on the wire in clear text.
|
token, and plain http would put it on the wire in clear text.
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
{
|
|
||||||
# Same reason the published one is asserted: a value the grammar
|
|
||||||
# rejects surfaces as `attempt to call elemAt on null` from inside
|
|
||||||
# the renderer, naming neither the option nor the offending value.
|
|
||||||
assertion = lib.all (t: parseScrape t != null) (lib.attrValues cfg.scrapeTargets);
|
|
||||||
message = ''
|
|
||||||
services.hyperhive.swarm.otel.scrapeTargets has ${
|
|
||||||
lib.concatMapStringsSep ", " (kv: "${kv.name} = \"${kv.value}\"") (
|
|
||||||
lib.filter (kv: parseScrape kv.value == null) (lib.attrsToList cfg.scrapeTargets)
|
|
||||||
)
|
|
||||||
}, which is not of the form <host>:<port>[/<path>][?<k>=<v>].
|
|
||||||
|
|
||||||
A trailing `?` with nothing after it is the usual cause. This
|
|
||||||
option is loopback-and-unauthenticated by contract, so it takes
|
|
||||||
no scheme and no credential.
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
# A job name used by BOTH scrape options. Within one option this
|
# A job name used by BOTH scrape options. Within one option this
|
||||||
# cannot happen — the module system refuses two definitions of the
|
# cannot happen — the module system refuses two definitions of the
|
||||||
|
|
@ -990,7 +927,10 @@ in
|
||||||
# and deploys perfectly while adding no data.
|
# and deploys perfectly while adding no data.
|
||||||
// lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
|
// lib.optionalAttrs (cfg.scrapeTargets != { } || cfg.publishedScrapeTargets != { }) {
|
||||||
prometheus.config.scrape_configs =
|
prometheus.config.scrape_configs =
|
||||||
lib.mapAttrsToList loopbackScrapeConfig cfg.scrapeTargets
|
lib.mapAttrsToList (job: target: {
|
||||||
|
job_name = job;
|
||||||
|
static_configs = [ { targets = [ target ]; } ];
|
||||||
|
}) cfg.scrapeTargets
|
||||||
# ⚠️ `++`, so the two kinds of target land in ONE list —
|
# ⚠️ `++`, so the two kinds of target land in ONE list —
|
||||||
# which is exactly why a job name may not appear in both
|
# which is exactly why a job name may not appear in both
|
||||||
# options. A list concatenation does not resolve a
|
# options. A list concatenation does not resolve a
|
||||||
|
|
|
||||||
|
|
@ -118,10 +118,6 @@ let
|
||||||
baoWithCollector = hive {
|
baoWithCollector = hive {
|
||||||
deploy.bao.enable = true;
|
deploy.bao.enable = true;
|
||||||
deploy.swarm-otel.enable = true;
|
deploy.swarm-otel.enable = true;
|
||||||
# A second job declared as bare `host:port`, so the pair of cases below
|
|
||||||
# reads one rendered scrape list: the store's entry carries a path, this
|
|
||||||
# one carries none.
|
|
||||||
swarm.otel.scrapeTargets.plain = "127.0.0.1:9999";
|
|
||||||
};
|
};
|
||||||
baoNoCollector = hive {
|
baoNoCollector = hive {
|
||||||
deploy.bao.enable = true;
|
deploy.bao.enable = true;
|
||||||
|
|
@ -137,14 +133,6 @@ let
|
||||||
# look there rather than at the host's service set.
|
# look there rather than at the host's service set.
|
||||||
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
|
baoUnits = machine: machine.containers.swarm-bao.config.systemd.services;
|
||||||
|
|
||||||
# The scrape list prometheus is handed, not the option a service declared:
|
|
||||||
# the address, the path and the query are one string on the way in and three
|
|
||||||
# fields on the way out, and only the second shape is what gets requested.
|
|
||||||
scrapeJob =
|
|
||||||
machine: job:
|
|
||||||
lib.findFirst (c: c.job_name == job) null
|
|
||||||
machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs;
|
|
||||||
|
|
||||||
# A priority collision is a property of the *option*, not
|
# A priority collision is a property of the *option*, not
|
||||||
# of the merged value's interior — nix throws the moment the value is
|
# of the merged value's interior — nix throws the moment the value is
|
||||||
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
|
# demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
|
||||||
|
|
@ -397,28 +385,6 @@ let
|
||||||
in
|
in
|
||||||
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
|
s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s";
|
||||||
}
|
}
|
||||||
{
|
|
||||||
# openbao serves no `/metrics` at all, so a scrape of the default path
|
|
||||||
# 404s: the store looks like a dead exporter, and every panel built on
|
|
||||||
# it renders empty rather than erroring.
|
|
||||||
name = "the store's scrape asks for the path openbao serves";
|
|
||||||
ok =
|
|
||||||
let
|
|
||||||
j = scrapeJob baoWithCollector "bao";
|
|
||||||
in
|
|
||||||
(j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Presence control for the case above: both fields are omitted rather
|
|
||||||
# than defaulted, so a target declared as bare `host:port` renders what
|
|
||||||
# it rendered before the path grammar existed.
|
|
||||||
name = "a target with no path renders neither metrics_path nor params";
|
|
||||||
ok =
|
|
||||||
let
|
|
||||||
j = scrapeJob baoWithCollector "plain";
|
|
||||||
in
|
|
||||||
j != null && !(j ? metrics_path) && !(j ? params);
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
# Absence arm. Unauthenticated by design, so it must not exist where
|
# Absence arm. Unauthenticated by design, so it must not exist where
|
||||||
# nothing reads it.
|
# nothing reads it.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue