export the swarm cache's size from atticd's database to the attic dashboard
This commit is contained in:
atlas
parent
99752f81d9
commit
c0429bf617
4 changed files with 291 additions and 1 deletions
|
|
@ -201,7 +201,7 @@ Grafana start — see [`credentials.md`](credentials.md).
|
|||
kept.
|
||||
|
||||
- `agents`: per-agent resource usage and Claude Code activity. The **activity** tab ends with a **Busiest agents** table: turns, input, output and cache-read tokens, and cost for each agent in the range, busiest first.
|
||||
- `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused. It reads the closure-pullable gauge and the `swarm-builder` and `hive-c0re` logs. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request, hit or storage panel. Filter by hive and agent.
|
||||
- `attic`: the swarm binary cache. Agents whose latest closure this hive can pull, and why not; the builder's pushes and push failures; closures hive-c0re pulled and deploy steps it refused; the cache's size and store-path count. It reads the closure-pullable gauge, the `swarm-builder` and `hive-c0re` logs, and the `hyperhive.attic.cache_bytes` and `hyperhive.attic.cache_nars` gauges, which a timer in the `swarm-attic` container (`swarm-attic-cache-size`) reads from atticd's database every five minutes and pushes through the container's forwarder. atticd itself exports no metrics and its logs don't reach the log store, so the board has no cache request or hit panel. Filter by hive and agent.
|
||||
- `authelia`, `bao`, `queue`: the identity provider, the secret store and the NATS queue.
|
||||
- `claude-usage`: Claude use by model, effort, token type and source. Filter by hive, service (agents or the review classifier) and agent. The service filter separates the classifier's API-key spend from the agents.
|
||||
- `egress`: what agents send through the egress proxy, in three tabs. **overview** has the totals (requests, agents active, refused, credential challenges, data transferred), requests over time by status, and the busiest agents and destinations. **refused** lists the hosts the proxy refused for each agent. **would-deny** lists what agents in `log` mode reached that restricted mode would refuse. The host lists are tables, one row per host (and agent), with a count column you can sort. Filter by agent.
|
||||
|
|
|
|||
|
|
@ -102,6 +102,16 @@ let
|
|||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there.
|
||||
forwarderTelemetryPort = 8903;
|
||||
# The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter
|
||||
# pushes to. Not 4318: the hive collector holds that in the host netns.
|
||||
forwarderOtlpPort = 8905;
|
||||
|
||||
# The exporter's unit and query. `V` is attic's stored value for a valid
|
||||
# chunk (`ChunkState::Valid`) and for a valid NAR (`NarState::Valid`).
|
||||
# `file_size` is the compressed size on disk, and each chunk is one row
|
||||
# however many NARs reference it.
|
||||
sizeUnit = "swarm-attic-cache-size";
|
||||
databaseUri = "file:${stateDir}/server.db?mode=ro";
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
# The cache's store identity, handed into the container as system credentials
|
||||
# for the forwarder's fetch unit. The host units keep reading the same files.
|
||||
|
|
@ -418,6 +428,7 @@ in
|
|||
clientId = forwarderClientId;
|
||||
tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
otlpMetricsPort = forwarderOtlpPort;
|
||||
store = {
|
||||
package = baoDeploy.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
|
|
@ -433,6 +444,86 @@ in
|
|||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||||
|
||||
# Reads atticd's database and pushes the cache's size to the
|
||||
# forwarder. It runs as atticd's dynamic user, shared by name, because
|
||||
# the state directory is mode 0700. With atticd stopped the name gets
|
||||
# another uid and the read fails. `-readonly` and `mode=ro` leave
|
||||
# the database to atticd, and `.timeout` waits out its writes.
|
||||
systemd.services.${sizeUnit} = {
|
||||
description = "push the swarm cache's size, read from atticd's database, to the swarm collector";
|
||||
wants = [ "${unit}.service" ];
|
||||
after = [
|
||||
"${unit}.service"
|
||||
"opentelemetry-collector.service"
|
||||
];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.sqlite
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = unit;
|
||||
DynamicUser = true;
|
||||
StateDirectory = stateName;
|
||||
SyslogIdentifier = sizeUnit;
|
||||
UMask = "0077";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
"AF_UNIX"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
query() {
|
||||
sqlite3 -readonly -cmd '.timeout 5000' ${lib.escapeShellArg databaseUri} "$1"
|
||||
}
|
||||
bytes="$(query "SELECT COALESCE(SUM(file_size), 0) FROM chunk WHERE state = 'V'")"
|
||||
nars="$(query "SELECT COUNT(*) FROM nar WHERE state = 'V'")"
|
||||
case "$bytes$nars" in
|
||||
"" | *[!0-9]*)
|
||||
echo "the database answered a non-number: bytes='$bytes' nars='$nars'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
now="$(date +%s%N)"
|
||||
gauge() {
|
||||
printf '{"name":"%s","gauge":{"dataPoints":[{"asInt":"%s","timeUnixNano":"%s"}]}}' "$1" "$2" "$now"
|
||||
}
|
||||
printf '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"%s"}}]},"scopeMetrics":[{"metrics":[%s,%s]}]}]}' \
|
||||
${lib.escapeShellArg machine} \
|
||||
"$(gauge hyperhive.attic.cache_bytes "$bytes")" \
|
||||
"$(gauge hyperhive.attic.cache_nars "$nars")" |
|
||||
curl --fail --silent --show-error --max-time 20 \
|
||||
-H 'Content-Type: application/json' --data-binary @- \
|
||||
http://127.0.0.1:${toString forwarderOtlpPort}/v1/metrics
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.${sizeUnit} = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2min";
|
||||
OnUnitActiveSec = "5min";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "attic binary cache server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
|
|||
|
|
@ -806,6 +806,178 @@
|
|||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Cache size",
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 53
|
||||
},
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "stat",
|
||||
"title": "Cache size",
|
||||
"description": "Compressed bytes of valid chunks, each counted once. Source: cache_bytes gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.attic.cache_bytes\"}[15m]))",
|
||||
"instant": true,
|
||||
"legendFormat": "size"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "stat",
|
||||
"title": "Store paths cached",
|
||||
"description": "Store paths (NARs) the cache holds as valid. Source: cache_nars gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.attic.cache_nars\"}[15m]))",
|
||||
"instant": true,
|
||||
"legendFormat": "store paths"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 33,
|
||||
"type": "timeseries",
|
||||
"title": "Cache size over time",
|
||||
"description": "Compressed bytes of valid chunks; garbage collection lowers it. Source: cache_bytes gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max({__name__=\"hyperhive.attic.cache_bytes\"})",
|
||||
"legendFormat": "size"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -497,6 +497,33 @@ let
|
|||
&& !(atticExternal.containers ? swarm-attic)
|
||||
&& atticExternal.services.hyperhive.swarm.otel.serviceProducers ? swarm-attic;
|
||||
}
|
||||
{
|
||||
# The exporter reads atticd's database through a read-only URI, as a
|
||||
# dynamic user and not root, and pushes to the forwarder's OTLP receiver,
|
||||
# whose port the forwarder opens. The host with the cache off is the
|
||||
# control that nothing of it renders there.
|
||||
name = "the cache's container exports its size from a read-only look at the database, as a non-root user, through the forwarder";
|
||||
ok =
|
||||
let
|
||||
inner = atticHere.containers.swarm-attic.config;
|
||||
svc = inner.systemd.services.swarm-attic-cache-size;
|
||||
f = inner.services.hyperhive.swarmJournalForwarder;
|
||||
port = toString f.otlpMetricsPort;
|
||||
in
|
||||
svc.serviceConfig.Type == "oneshot"
|
||||
&& svc.serviceConfig.DynamicUser
|
||||
&& svc.serviceConfig.User == "swarm-attic"
|
||||
&& lib.hasInfix "-readonly" svc.script
|
||||
&& lib.hasInfix "server.db?mode=ro" svc.script
|
||||
&& !(lib.hasInfix "mode=rw" svc.script)
|
||||
&& lib.hasInfix "state = 'V'" svc.script
|
||||
&& lib.hasInfix "hyperhive.attic.cache_bytes" svc.script
|
||||
&& lib.hasInfix "hyperhive.attic.cache_nars" svc.script
|
||||
&& lib.hasInfix "127.0.0.1:${port}/v1/metrics" svc.script
|
||||
&& f.otlpMetricsPort != null
|
||||
&& inner.systemd.timers ? swarm-attic-cache-size
|
||||
&& !(atticExternal.containers ? swarm-attic);
|
||||
}
|
||||
{
|
||||
# attic checks its own tokens and a nix client cannot follow a login
|
||||
# redirect, so the vhost carries no `auth_request`; the tokens ride in a
|
||||
|
|
|
|||
Loading…
Reference in a new issue