export the swarm cache's size from atticd's database to the attic dashboard
This commit is contained in:
atlas
parent
99752f81d9
commit
c0429bf617
3 changed files with 291 additions and 1 deletions
|
|
@ -102,6 +102,16 @@ let
|
|||
forwarderUnit = "swarm-journal-forwarder-oidc";
|
||||
# In the host netns: distinct from every other collector's there.
|
||||
forwarderTelemetryPort = 8903;
|
||||
# The forwarder's OTLP/HTTP metrics receiver, which the cache-size exporter
|
||||
# pushes to. Not 4318: the hive collector holds that in the host netns.
|
||||
forwarderOtlpPort = 8905;
|
||||
|
||||
# The exporter's unit and query. `V` is attic's stored value for a valid
|
||||
# chunk (`ChunkState::Valid`) and for a valid NAR (`NarState::Valid`).
|
||||
# `file_size` is the compressed size on disk, and each chunk is one row
|
||||
# however many NARs reference it.
|
||||
sizeUnit = "swarm-attic-cache-size";
|
||||
databaseUri = "file:${stateDir}/server.db?mode=ro";
|
||||
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
|
||||
# The cache's store identity, handed into the container as system credentials
|
||||
# for the forwarder's fetch unit. The host units keep reading the same files.
|
||||
|
|
@ -418,6 +428,7 @@ in
|
|||
clientId = forwarderClientId;
|
||||
tokenUrl = "${toString autheliaCfg.url}/api/oidc/token";
|
||||
telemetryPort = forwarderTelemetryPort;
|
||||
otlpMetricsPort = forwarderOtlpPort;
|
||||
store = {
|
||||
package = baoDeploy.package;
|
||||
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
|
|
@ -433,6 +444,86 @@ in
|
|||
};
|
||||
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames forwarderStoreFiles;
|
||||
|
||||
# Reads atticd's database and pushes the cache's size to the
|
||||
# forwarder. It runs as atticd's dynamic user, shared by name, because
|
||||
# the state directory is mode 0700. With atticd stopped the name gets
|
||||
# another uid and the read fails. `-readonly` and `mode=ro` leave
|
||||
# the database to atticd, and `.timeout` waits out its writes.
|
||||
systemd.services.${sizeUnit} = {
|
||||
description = "push the swarm cache's size, read from atticd's database, to the swarm collector";
|
||||
wants = [ "${unit}.service" ];
|
||||
after = [
|
||||
"${unit}.service"
|
||||
"opentelemetry-collector.service"
|
||||
];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.sqlite
|
||||
];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = unit;
|
||||
DynamicUser = true;
|
||||
StateDirectory = stateName;
|
||||
SyslogIdentifier = sizeUnit;
|
||||
UMask = "0077";
|
||||
CapabilityBoundingSet = [ "" ];
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectSystem = "strict";
|
||||
ProtectHome = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectControlGroups = true;
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
"AF_UNIX"
|
||||
];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
LockPersonality = true;
|
||||
SystemCallArchitectures = "native";
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
query() {
|
||||
sqlite3 -readonly -cmd '.timeout 5000' ${lib.escapeShellArg databaseUri} "$1"
|
||||
}
|
||||
bytes="$(query "SELECT COALESCE(SUM(file_size), 0) FROM chunk WHERE state = 'V'")"
|
||||
nars="$(query "SELECT COUNT(*) FROM nar WHERE state = 'V'")"
|
||||
case "$bytes$nars" in
|
||||
"" | *[!0-9]*)
|
||||
echo "the database answered a non-number: bytes='$bytes' nars='$nars'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
now="$(date +%s%N)"
|
||||
gauge() {
|
||||
printf '{"name":"%s","gauge":{"dataPoints":[{"asInt":"%s","timeUnixNano":"%s"}]}}' "$1" "$2" "$now"
|
||||
}
|
||||
printf '{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"%s"}}]},"scopeMetrics":[{"metrics":[%s,%s]}]}]}' \
|
||||
${lib.escapeShellArg machine} \
|
||||
"$(gauge hyperhive.attic.cache_bytes "$bytes")" \
|
||||
"$(gauge hyperhive.attic.cache_nars "$nars")" |
|
||||
curl --fail --silent --show-error --max-time 20 \
|
||||
-H 'Content-Type: application/json' --data-binary @- \
|
||||
http://127.0.0.1:${toString forwarderOtlpPort}/v1/metrics
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.${sizeUnit} = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "2min";
|
||||
OnUnitActiveSec = "5min";
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.${unit} = {
|
||||
description = "attic binary cache server";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
|
|
|||
|
|
@ -806,6 +806,178 @@
|
|||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Cache size",
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 53
|
||||
},
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "stat",
|
||||
"title": "Cache size",
|
||||
"description": "Compressed bytes of valid chunks, each counted once. Source: cache_bytes gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.attic.cache_bytes\"}[15m]))",
|
||||
"instant": true,
|
||||
"legendFormat": "size"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "stat",
|
||||
"title": "Store paths cached",
|
||||
"description": "Store paths (NARs) the cache holds as valid. Source: cache_nars gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max(last_over_time({__name__=\"hyperhive.attic.cache_nars\"}[15m]))",
|
||||
"instant": true,
|
||||
"legendFormat": "store paths"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": {
|
||||
"mode": "fixed",
|
||||
"fixedColor": "blue"
|
||||
},
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "none",
|
||||
"colorMode": "value",
|
||||
"textMode": "value",
|
||||
"justifyMode": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 33,
|
||||
"type": "timeseries",
|
||||
"title": "Cache size over time",
|
||||
"description": "Compressed bytes of valid chunks; garbage collection lowers it. Source: cache_bytes gauge.",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 54
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "@datasourceUid@"
|
||||
},
|
||||
"expr": "max({__name__=\"hyperhive.attic.cache_bytes\"})",
|
||||
"legendFormat": "size"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 1,
|
||||
"fillOpacity": 10,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"axisSoftMin": 0
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"calcs": []
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -497,6 +497,33 @@ let
|
|||
&& !(atticExternal.containers ? swarm-attic)
|
||||
&& atticExternal.services.hyperhive.swarm.otel.serviceProducers ? swarm-attic;
|
||||
}
|
||||
{
|
||||
# The exporter reads atticd's database through a read-only URI, as a
|
||||
# dynamic user and not root, and pushes to the forwarder's OTLP receiver,
|
||||
# whose port the forwarder opens. The host with the cache off is the
|
||||
# control that nothing of it renders there.
|
||||
name = "the cache's container exports its size from a read-only look at the database, as a non-root user, through the forwarder";
|
||||
ok =
|
||||
let
|
||||
inner = atticHere.containers.swarm-attic.config;
|
||||
svc = inner.systemd.services.swarm-attic-cache-size;
|
||||
f = inner.services.hyperhive.swarmJournalForwarder;
|
||||
port = toString f.otlpMetricsPort;
|
||||
in
|
||||
svc.serviceConfig.Type == "oneshot"
|
||||
&& svc.serviceConfig.DynamicUser
|
||||
&& svc.serviceConfig.User == "swarm-attic"
|
||||
&& lib.hasInfix "-readonly" svc.script
|
||||
&& lib.hasInfix "server.db?mode=ro" svc.script
|
||||
&& !(lib.hasInfix "mode=rw" svc.script)
|
||||
&& lib.hasInfix "state = 'V'" svc.script
|
||||
&& lib.hasInfix "hyperhive.attic.cache_bytes" svc.script
|
||||
&& lib.hasInfix "hyperhive.attic.cache_nars" svc.script
|
||||
&& lib.hasInfix "127.0.0.1:${port}/v1/metrics" svc.script
|
||||
&& f.otlpMetricsPort != null
|
||||
&& inner.systemd.timers ? swarm-attic-cache-size
|
||||
&& !(atticExternal.containers ? swarm-attic);
|
||||
}
|
||||
{
|
||||
# attic checks its own tokens and a nix client cannot follow a login
|
||||
# redirect, so the vhost carries no `auth_request`; the tokens ride in a
|
||||
|
|
|
|||
Loading…
Reference in a new issue