vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity
hyperhive/nix/host-modules/swarm-grafana.nix
atlas b072fbf4d6 push ci run metrics to victoriametrics, with a grafana board 
A swarm-ci-metrics host unit on a 5-minute timer lists every forge
repo's Actions runs as the new read-only ci-metrics account and pushes
each finished run's duration and queue wait over OTLP, stamped at the
run's stop time. The first pass backfills from run 1. swarm-controller
mints the account's token into bao and keeps a ci-metrics team with
read on the Actions unit in every organisation.

Closes #5376
2026-10-16 20:43:11 +02:00

1602 lines
75 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# The swarm's metrics UI: one Grafana for the whole swarm, in a
# `swarm-grafana` nixos-container, reading the VictoriaMetrics store.
#
# Two containers rather than one, on the operator's call: Grafana can be
# restarted, reconfigured or broken without taking the TSDB down with it.
# They are a pair, not a unit.
#
# Dashboards are provisioned from files in this repo, deliberately not created
# in the UI: the store path is immutable, so Grafana is told not to accept UI
# edits it would lose on the next rebuild.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.grafana;
deployCfg = config.services.hyperhive.deploy;
networkCfg = config.services.hyperhive.network;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
autheliaDomain = hyperhiveCfg.swarm.authelia.domain;
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = deployCfg.bao;
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib gatewayCfg;
tlsCfg = deployCfg.hive-controller.tls;
};
# Spelled ONCE. The provisioned datasource declares it and every panel in the
# provisioned dashboards references it; a second literal would be free to
# drift, and the failure is a dashboard that renders with empty panels rather
# than an error.
datasourceUid = "swarm-victoriametrics";
# The logs store's own uid. Separate from the metrics one and spelled once for
# the same reason: a dashboard panel names it, and a second literal would be
# free to drift into a panel that renders empty rather than erroring.
logsDatasourceUid = "swarm-victorialogs";
# Which field carries the log level, told to the READER because the writer
# cannot be told: VictoriaLogs' OTLP ingester names the stored field
# `severity_text` unconditionally, with no option to rename it, but
# Grafana's log-level buttons filter on a field called `level`. So the
# mapping is made here, one `logLevelRules` entry per level below.
#
# 🔑 A rule wrong in any of three ways provisions cleanly and its level
# button silently returns zero rows — `enabled` must be literally `true`,
# `level` must be a canonical Grafana `LogLevel` (not an alias like `warn`
# or `fatal`), and `value` must match ../journald-severity.nix's uppercase
# STORED text, not its lowercase keys. `Unspecified` is deliberately left
# unmapped so the "no severity" panel keeps counting it. Full walkthrough
# (why each rule shape is what it is, the query a level button actually
# emits): docs/scheduler/observability.md#why-explores-level-buttons-need-the-datasource-told
logLevelRules =
let
fromSeverityText = value: level: {
field = "severity_text";
operator = "equals";
enabled = true;
inherit value level;
};
in
[
(fromSeverityText "FATAL" "critical")
(fromSeverityText "ERROR" "error")
(fromSeverityText "WARN" "warning")
(fromSeverityText "INFO" "info")
(fromSeverityText "DEBUG" "debug")
(fromSeverityText "TRACE" "trace")
];
# An alert rule that fires while a LogsQL stats query counts more than zero
# lines. `expr` carries its own `_time:` window, so the count covers exactly
# that span; `relativeTimeRange` matches it so the rule editor's preview
# queries the same span.
#
# No lines is the healthy state, so no data is `OK`. A query that fails is
# `Error`, which the rule list shows separately from both.
logCountRule =
{
uid,
title,
windowSeconds,
expr,
for,
summary,
}:
{
inherit uid title for;
condition = "C";
data = [
{
refId = "A";
relativeTimeRange = {
from = windowSeconds;
to = 0;
};
datasourceUid = logsDatasourceUid;
model = {
refId = "A";
datasource = {
type = "victoriametrics-logs-datasource";
uid = logsDatasourceUid;
};
queryType = "stats";
inherit expr;
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
refId = "B";
type = "reduce";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
refId = "C";
type = "threshold";
expression = "B";
conditions = [
{
evaluator = {
type = "gt";
params = [ 0 ];
};
}
];
};
}
];
noDataState = "OK";
execErrState = "Error";
annotations = { inherit summary; };
};
# `logCountRule` inverted: fires while the count is below one. `expr` must
# be an ungrouped `stats count()`, which answers 0 rather than no row when
# nothing matched; a grouped count drops the group instead, and a dropped
# instance resolves rather than fires. No data is `Alerting` so a query
# that returns nothing at all still fires.
logAbsenceRule =
args:
let
rule = logCountRule args;
in
rule
// {
data = map (
query:
if query.refId == "C" then
lib.recursiveUpdate query {
model.conditions = [
{
evaluator = {
type = "lt";
params = [ 1 ];
};
}
];
}
else
query
) rule.data;
noDataState = "Alerting";
};
# The shipped dashboards carry `@datasourceUid@` / `@logsDatasourceUid@` where
# a real deployment needs the uids above. They are substituted here rather
# than committed with the literals so the single binding stays single.
renderDashboard =
name:
pkgs.writeText name (
builtins.replaceStrings
[ "@datasourceUid@" "@logsDatasourceUid@" ]
[ datasourceUid logsDatasourceUid ]
(builtins.readFile (./swarm-grafana/dashboards + "/${name}"))
);
# Grafana's file provider wants a DIRECTORY to scan, so the rendered files are
# collected into one.
dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" (
map
(name: {
inherit name;
path = renderDashboard name;
})
[
"agents.json"
"authelia.json"
"bao.json"
"ci.json"
"claude-usage.json"
"collector.json"
"controller.json"
"forge.json"
"gateway.json"
"logstore.json"
"metricstore.json"
"queue.json"
]
);
# A reader of the store is defined by holding a certificate the store
# accepts, never by standing next to it — the rule
# ./glue-queue-agent-credential.nix states in full.
#
# Also asserted rather than gating anything, and that is the ruling this
# module's second delivery route was deleted under: the store exists so that
# ONE secret is placed out of band per host — this certificate — and every
# other secret is read with it. A Grafana host without it has not been given
# its identity yet, which is a thing to say out loud rather than to route
# around by reaching into authelia's tree whenever it happens to be local.
#
# 🩸 Grafana's OWN leaf, not `clientCertFile` — the hive's. Bao matches a
# cert-auth role on the CN, so one leaf shared by four readers would be ONE
# principal holding the union of four grants: this unit could read every
# agent credential in the swarm, when what it needs is the one path
# `storeSecretPath` names.
haveClientIdentity =
baoDeploy.grafanaOidcClientCertFile != null && baoDeploy.grafanaOidcClientKeyFile != null;
autheliaUrl = toString hyperhiveCfg.swarm.authelia.url;
# The container's journal forwarder pushes to its own receiver on the swarm
# collector, the route `serviceProducers.swarm-grafana` in
# ./swarm-otel-service.nix declares, as a client of its own rather than the
# browser login's. The port suffix is elided on 443, as in ./swarm-bao.nix.
forwarderClientId = cfg.otel.clientId;
gatewayPortSuffix = lib.optionalString (
gatewayCfg.httpsPort != 443
) ":${toString gatewayCfg.httpsPort}";
forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
forwarderUnit = "swarm-journal-forwarder-oidc";
# In the host netns: distinct from every other collector's there, which the
# assertion below checks.
forwarderTelemetryPort = 8896;
# Grafana's store identity, handed into the container as system credentials
# (credential id → host source) for the forwarder's fetch unit. The host units
# below keep reading the same files directly.
storeFiles = lib.filterAttrs (_: source: source != null) {
grafana-bao-cert = baoDeploy.grafanaOidcClientCertFile;
grafana-bao-key = baoDeploy.grafanaOidcClientKeyFile;
grafana-bao-server-ca = baoDeploy.serverCaFile;
};
# nixos-container expands `extraFlags` unquoted, so a source path must be
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
# The three secrets below land on the host's /run (tmpfs), which the
# container bind-mounts read-only at the same path: none reaches persistent
# disk, and a reboot fetches them again. Each file is absent until its
# delivery unit has run, and Grafana does not start without them — `$__file{}`
# on a missing path is a startup error, and an empty admin password would
# fall back to the upstream default. A delivery unit that writes a file
# restarts or starts Grafana (`refresh_consumer`).
#
# Spelled once: the unit that writes a file and the config that names it are
# a few hundred lines apart, and a Grafana reading a path nothing writes is
# a login that fails with nothing in any log about the file.
secretDir = "/run/swarm-grafana-secrets";
secretPath = "${secretDir}/${cfg.oidc.clientId}.secret";
# Where the three files sat on persistent disk before they moved to /run.
legacyHostSecretPaths = map (p: "/var/lib/nixos-containers/${cfg.machine}${p}") [
"/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"
"/var/lib/grafana-admin/password"
"/var/lib/grafana-metrics/bearer"
];
# Where the publisher on authelia's host leaves this client's secret. A
# swarm service's client is registered once for the whole swarm, so its
# secret is one value and not one per hive. Both ends compose this from the
# same swarm-wide client id, so a rename cannot leave one of them behind.
#
# ⚠️ The prefix is not decoration: it is what this host's read grant covers,
# so a path outside it answers 403 rather than a miss, however correct it
# looks.
storeSecretPath = "secret/swarm/services/${cfg.oidc.clientId}/oidc/client";
# Grafana's own datasource-encryption key. Generated in-container (see the
# unit below) because nothing outside the container ever reads it — unlike
# the OIDC secret above, whose other reader is authelia's container.
secretKeyPath = "/var/lib/grafana-secret/secret_key";
# The built-in `admin` account's password. Grafana's API takes basic auth
# whatever the login form says, so this account is reachable from the public
# vhost and its password cannot be the upstream default. Minted once into the
# store by `swarm-bao-grafana-admin` below, which `swarm-bao.nix`'s
# `grafanaOidcReaders` grants create and read on this one leaf.
adminPasswordStorePath = "secret/swarm/services/${cfg.oidc.clientId}/admin/password";
adminPasswordPath = "${secretDir}/admin-password";
# The bearer the metrics datasource presents at the metrics store's query
# API, which `swarm-metrics-auth` on the store's host checks against the
# same store path, and the logs datasource at the log store's query API,
# which a second `swarm-metrics-auth` instance in ./swarm-victorialogs.nix
# checks against it. Minted once by `swarm-bao-grafana-metrics-bearer`
# below, under the same grant as the admin password. The file holds the whole
# header value, `Bearer <value>`: nixpkgs accepts a `secureJsonData` entry
# only when it is exactly one `$__file{}`.
metricsBearerStorePath = "secret/swarm/services/${cfg.oidc.clientId}/metrics/bearer";
metricsBearerPath = "${secretDir}/metrics-bearer";
# swarm-controller's token for Grafana's API, minted by
# `swarm-bao-grafana-controller-token` below for a Viewer service account of
# this name. `controllerGrafanaTokenLeaf` in ./swarm-bao.nix is the nix half
# of the path, `swarm_secret_client::grafana::controller_token_path` the Rust
# half.
controllerAccountName = "swarm-controller";
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/grafana/token";
socketPath = "${deployCfg.grafana.socketDir}/grafana.sock";
# Both static NixOS ids, and both checked rather than assumed: nginx has
# `ids.gids.nginx = 60`, Grafana has `ids.uids.grafana = 196` — but there is
# deliberately no `ids.gids.grafana`, which is why the OIDC-secret unit
# below lands its file root-group at 0400 instead of reaching for one.
nginxGid = config.ids.gids.nginx;
grafanaUid = config.ids.uids.grafana;
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
# First statement of each delivery script, ahead of the store login: the
# directory is the container's bind source, so it has to exist even when the
# store does not answer. tmpfiles creates it too, but unordered with the
# container on a switch.
ensureSecretDir = "install -d -m 0755 ${lib.escapeShellArg secretDir}";
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
in
{
# What the service IS to every hive is `swarm.grafana` in
# ./swarm-grafana-service.nix. What the host running it decides is here:
# which build it runs, where its datasources point, which plugins sit in its
# store path, and the directory it shares a socket with nginx through.
# `enable` already lives in ./deploy.nix, which also carries the renames.
#
# Both datasource URLs are wiring, so they live here.
options.services.hyperhive.deploy.grafana = {
package = lib.mkOption {
type = lib.types.package;
default = pkgs.grafana;
defaultText = lib.literalExpression "pkgs.grafana";
description = "Grafana package to run.";
};
socketDir = lib.mkOption {
type = lib.types.str;
default = "/run/swarm-grafana";
readOnly = true;
description = ''
Directory holding the unix socket Grafana serves on, shared between
the host (where nginx runs) and the container (where Grafana runs).
Fixed, not configurable: the directory's mode is what keeps the socket
private, so it has to be dedicated to this one socket, and a settable
path could be pointed at a directory that carries something else.
⚠️ **Grafana takes no TCP port at all, and that is the point.** Every
swarm service container shares the host's network namespace, so a
port is a swarm-wide resource that two modules can silently both
claim — which is exactly what happened: Grafana defaulted to
upstream's 3000, so does the forge, and `grafana.<swarm-domain>`
served the forge with no bind error and nothing in any log.
A socket has a path, and a path collision is a build-time conflict
rather than a runtime coin toss.
'';
};
datasourceUrl = lib.mkOption {
type = lib.types.str;
default = "https://${hyperhiveCfg.swarm.victoriametrics.domain}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.victoriametrics.domain}"'';
description = ''
Where the provisioned datasource points. Defaults to the metrics
store's swarm name, served by the gateway of whichever host runs the
store, so Grafana and the store need not share a host.
The datasource sends the bearer `swarm-bao-grafana-metrics-bearer`
keeps in the swarm secret store, which is what the store's query
routes on that vhost check. Whatever URL this names receives that
bearer.
'';
};
logsDatasourceUrl = lib.mkOption {
type = lib.types.str;
default = "https://${hyperhiveCfg.swarm.victorialogs.domain}";
defaultText = lib.literalExpression ''"https://''${services.hyperhive.swarm.victorialogs.domain}"'';
description = ''
Where the provisioned logs datasource points. Defaults to the log
store's swarm name, served by the gateway of whichever host runs the
store, so Grafana and the store need not share a host.
Provisioned unconditionally, like the metrics datasource — the store
being off is a deployment choice rather than a reason to withhold the
connection, and an operator whose logs live elsewhere sets this.
The datasource sends the same bearer as the metrics one, which the
store's `/select/logsql/` route checks. Whatever URL this names
receives that bearer.
'';
};
plugins = lib.mkOption {
type = lib.types.listOf lib.types.package;
default = [
pkgs.grafanaPlugins.grafana-metricsdrilldown-app
# Reading the swarm's logs needs the datasource that speaks LogsQL.
# Nothing is paired with it to browse them: Explore is that browser.
pkgs.grafanaPlugins.victoriametrics-logs-datasource
];
defaultText = lib.literalExpression ''
[
pkgs.grafanaPlugins.grafana-metricsdrilldown-app
pkgs.grafanaPlugins.victoriametrics-logs-datasource
]'';
example = lib.literalExpression "[ pkgs.grafanaPlugins.grafana-piechart-panel ]";
description = ''
Grafana plugins to install, as packages. Declarative rather than
installed through the UI, which is the only shape that works here:
plugin management is **server-admin** scoped, and on an SSO hive
nobody holds that role — `users.auto_assign_org_role` grants an
*org* role, and the built-in local admin that does hold server
admin cannot log in because the login form is disabled whenever
SSO is configured.
That is a deliberate pair of decisions rather than an oversight,
and this option is the way through it: plugins live in the store
and in git, so they survive a container rebuild and a state reset,
and the container needs no runtime egress to grafana.com.
⚠️ This list is the ONLY way a plugin reaches this Grafana.
Setting it at all disables upstream's `preinstall` mechanism (see
the `declarativePlugins` comment below), which is what normally
fetches the Drilldown apps on first start — so a plugin left off
this list is absent, not merely un-pinned.
Metrics Drilldown is on by default because this swarm's Grafana
exists to read VictoriaMetrics, and the queryless metrics browser
is the part of that people actually reach for.
Its Logs / Traces / Profiles siblings are deliberately NOT here,
and for two different reasons. Traces and Profiles front Tempo
and Pyroscope, which this swarm does not run — a UI for a backend
that does not answer. Logs Drilldown is the subtler one: this
swarm *does* run a log store, so that argument does not apply
to it, but the app is Loki-only and no configuration here changes
that. Its volume views call `/loki/api/v1/index/volume`, which
VictoriaLogs does not implement and answers "unsupported path
requested". Logs are browsed through Explore instead.
'';
};
};
config = lib.mkIf deployCfg.grafana.enable {
# The gateway name, inside `deploy.grafana` — that
# guard is the load-bearing part. Every hive in a swarm may know this UI
# exists, but only the host that RUNS it may claim the name; a client
# hive declaring the vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
# This host serves the vhost, and the container behind it resolves
# through the hive's dnsmasq.
services.hyperhive.gateway.enable = lib.mkDefault true;
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
# Registering the client is NOT here: it has to happen on the host that
# runs authelia, and this whole block is gated on the host that runs
# Grafana. ./glue-grafana-oidc-client.nix does it.
# This arm fires only where Grafana runs — this whole block is under
# `deploy.grafana.enable`. SSO itself needs no assertion:
# `swarm.authelia.url` is non-nullable, so every swarm has one and the
# OIDC block below always names a real endpoint.
assertions = [
{
# The other half of one login: the OIDC block names a `$__file{}` that
# `swarm-bao-grafana-oidc.service` below writes, and that unit reads
# the store with this host's client certificate. No certificate, no
# secret, and the same no-way-in Grafana as the arm above.
assertion = haveClientIdentity;
message = ''
services.hyperhive.deploy.grafana.enable requires this host to hold a
swarm-secret-store client identity: set both
services.hyperhive.deploy.bao.grafanaOidcClientCertFile
services.hyperhive.deploy.bao.grafanaOidcClientKeyFile
Grafana's OIDC client secret is minted by authelia and read out of
the store, on every host that runs Grafana — including the host that
runs authelia. That is one delivery route rather than two, and it is
what the store is for: a certificate is the credential placed out of
band, and every other secret comes from the store with it.
⚠️ Grafana's OWN leaf, not deploy.bao.clientCertFile. That one is the
hive's, and its grant reads every agent credential in the swarm and
this hive's own secrets; this role reads the one path Grafana's
client secret lives at. Pointing this option at the hive's leaf would
evaluate, deploy and log in — and undo the split.
On a hive that runs the store, glue-bao-tls.nix supplies both as
defaults and there is nothing to do. Elsewhere the leaf is issued
from that CA out of band and named here — see docs/swarm/secrets.md.
'';
}
{
assertion = lib.all oneWord (lib.attrValues storeFiles);
message = ''
Grafana's store identity paths must not contain whitespace: nspawn's
--load-credential receives them through nixos-container's unquoted
extra flags.
'';
}
{
assertion =
(hyperhiveCfg.swarm.otel.serviceProducers.${cfg.machine}.clientId or null) == forwarderClientId;
message = ''
services.hyperhive.swarm.otel.serviceProducers has no entry
"${cfg.machine}" with clientId "${forwarderClientId}", so the swarm
collector has no receiver for Grafana's journal forwarder. Declare
it in swarm-otel-service.nix.
'';
}
{
assertion =
!(lib.elem forwarderTelemetryPort [
hyperhiveCfg.otel.telemetryPort
hyperhiveCfg.swarm.otel.telemetryPort
hyperhiveCfg.swarm.bao.otel.telemetryPort
hyperhiveCfg.swarm.nats.otel.telemetryPort
]);
message = ''
Grafana's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
collides with another collector's in the host netns: the hive
collector's, the swarm collector's, the store forwarder's or the
queue forwarder's.
'';
}
];
# Websockets: Grafana Live streams panel updates over one, and without
# the upgrade headers dashboards load and then never refresh — which
# reads as stale data rather than as a proxy fault.
services.hyperhive.gateway.vhosts."${cfg.domain}" = {
# Grafana serves its Prometheus exposition to anyone, signed in or not
# (`metrics.enabled` below, no `basic_auth_*`), so this location's auth
# is the only gate on it. `=` is enough: nginx matches the decoded,
# normalised path, so `//metrics`, `/./metrics`, `/%6detrics` and
# `/metrics?x` all land here, and Grafana itself answers `/metrics/` with
# a login redirect.
#
# The collector requests exactly this URL, declared as
# `publishedScrapeTargets.grafana` in ./swarm-otel-service.nix; the two
# must agree, since that string is also the token's audience. Authelia's
# rules for it admit the collector's client and deny every other subject
# (`metricsRules` in ./swarm-authelia.nix).
locations."= /metrics" = {
proxyPass = "http://unix:${socketPath}:/metrics";
extraConfig = ''
auth_request /__grafana_metrics_authz;
'';
};
# The subrequest, same shape as the forge's `= /__forge_metrics_authz`
# (./hive-forge/default.nix): no `error_page 401 =302`, so a scraper
# gets the 401 rather than a login page parsed as metrics.
locations."= /__grafana_metrics_authz" = {
proxyPass = "https://${autheliaDomain}/api/authz/auth-request";
# nixpkgs appends its OWN `Host $host` after extraConfig, which would
# override verifiedProxyTo's — see hive-gateway/vhost-lib.nix.
recommendedProxySettings = false;
extraConfig = ''
internal;
${gatewayCfg.lib.verifiedProxyTo autheliaDomain}
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-Method $request_method;
proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
locations."/" = {
proxyPass = "http://unix:${socketPath}:/";
proxyWebsockets = true;
extraConfig = ''
# Grafana builds its OAuth redirect from the ORIGINAL request.
# Without these every request looks like it arrived at 127.0.0.1
# over plain http, and the redirect sent to authelia names a
# host the browser cannot reach.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
};
# Order the container after the host CA service so the bind source below
# exists before nspawn sets the mount up. nspawn also fails to start it
# when a credential source is missing, so where this host mints the leaf
# the container pulls the minting unit in and waits for it, as
# ./swarm-builder.nix's does.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];
}
];
# The socket directory, created host-side before the container starts.
# nixos-container refuses to start when a bind source is missing, so this
# rule is a prerequisite of the mount rather than a tidiness measure.
#
# Owned by Grafana's uid so it can create the socket, group nginx so the
# gateway can traverse; `0750` keeps everything else out. Numeric ids
# because the host has no `grafana` account — Grafana lives in the
# container, and only the number crosses that boundary.
#
# The secrets directory is the container's read-only bind source, so it
# has to exist before the container starts even when the store does not
# answer. The delivery units create it before their store login too
# (`ensureSecretDir`), which covers a switch, where this rule is unordered
# with the container restart. 0755 because Grafana reads the files as its
# own uid; each file is 0400 and owned by it. The legacy files are removed
# so no superseded secret stays behind on the host's disk.
systemd.tmpfiles.rules = [
"d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -"
"d ${secretDir} 0755 root root -"
]
++ map (p: "r ${p}") legacyHostSecretPaths;
# THE delivery unit — one route, in every deployment. The secret authelia
# minted arrives out of the swarm secret store, which the publisher on
# authelia's host wrote it into, and that is true whether authelia is a
# network away or in the container next door.
#
# 🩸 No second unit copies the plaintext out of authelia's host tree
# where the two are co-located. The store exists so a host holds ONE
# out-of-band secret — its client certificate — and reads everything else
# with it, so skipping the store when the producer is local would buy a
# round trip and cost a second delivery path, a second way for the file
# to be wrong, and a gate whose false arm is a Grafana with no login.
# Recorded in docs/swarm/secrets.md.
#
# Shaped after ./glue-queue-agent-credential.nix, because it is a store
# reader and this is what the store's readers do: a cert login that fails
# LOUDLY, since every state it fails on is one a retry fixes, then a read
# that degrades QUIETLY, since no retry turns "no value there" into a value.
#
# ⚠️ An ABSENT secret is "not published yet", not a fault of this host's.
# The publisher runs on the authelia host and authelia mints on its first
# boot, so early in a swarm's life there is legitimately nothing to read.
# Nothing here writes a stand-in: there is no such thing as a locally valid
# OIDC client secret, and a placeholder would turn a Grafana that cannot
# log anyone in into one whose token exchange is refused.
#
# ⇒ So the absence surfaces at Grafana rather than here, and this unit's job
# is to leave the journal line that says which of the three it was: the
# secret not arriving is a Grafana nobody can log into, and a named cause is
# the only thing separating that from a mystery.
systemd.services.swarm-bao-grafana-oidc = {
description = "fetch Grafana's OIDC client secret from the swarm secret store";
# Ordered after the store where it runs here
# (./glue-bao-readers-store-order.nix); elsewhere the timeout below
# bounds the attempt.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.systemd
];
# ./lib/store-retry.nix. Grafana's container is ordered after this unit
# and waits for one attempt; a secret a later attempt lands restarts
# Grafana (below).
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
# No `RemainAfterExit`: the timer below starts this again, and
# starting an active unit is a no-op.
Type = "oneshot";
SyslogIdentifier = "swarm-bao-grafana-oidc";
# What actually bounds each attempt below. Stated here rather than left
# to systemd's default, so the number a boot waits on is in the file
# that waits.
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
# Absent means the system trust store, which is what a deployment with a
# real CA wants and what a self-signed one must not be left with.
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${ensureSecretDir}
# `bao`'s own message is the only thing separating a missing value from
# a refused identity from an unreachable host. This unit's degraded
# mode is correct for all three, so it reports which one rather than
# asserting all three in a sentence of ours.
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# Cert auth is a login, not a transport setting. The `BAO_CLIENT_*`
# variables above only decide which certificate the TLS handshake
# presents; without a token `bao` asks its token helper instead, and
# that is a `sh` this unit's `path` does not carry. `-token-only`
# answers on stdout and skips the helper on both sides.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with this host's certificate; leaving Grafana's OIDC client secret as it is." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
if ! secret="$(bao kv get -field=value ${lib.escapeShellArg storeSecretPath} 2>"$err")"; then
echo "swarm-bao did not return ${storeSecretPath}; Grafana has no OIDC client secret yet." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 0
fi
if [ -z "$secret" ]; then
echo "swarm-bao returned an empty ${storeSecretPath}; leaving the file as it is." >&2
exit 0
fi
# Written with a shell builtin, never handed to a program: `printf` is
# bash's own, so the plaintext never becomes an argument in /proc the
# way `install <<<"$secret"` or an `echo` from `path` would.
#
# Owned by Grafana's own uid, unlike the matrix sibling which lands
# root-owned: tuwunel's secret is read by `LoadCredential` as root
# before the sandbox exists, whereas Grafana expands `$__file{}`
# itself, as itself, while parsing its config. This container sets no
# `privateUsers`, so the host uid is the container uid, and both sides
# take it from the same static NixOS id.
#
# Group is root, not grafana, and that is forced rather than chosen:
# `ids.uids.grafana` is a static id but there is no `ids.gids.grafana`
# — the group's gid is allocated at activation inside the container, so
# the host cannot know it at eval time. Harmless here because 0400
# grants the group nothing; if this mode ever widens, the gid has to be
# discovered at runtime rather than assumed.
if secret_differs ${lib.escapeShellArg secretPath} "$secret"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg secretPath} "$secret"
fi
# `$__file{}` is expanded once, while Grafana parses its config.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg secretPath}
'';
};
systemd.timers.swarm-bao-grafana-oidc = storeRefreshTimer "re-read Grafana's OIDC client secret from the swarm secret store";
# The admin password: created in the store if it holds none, then copied
# into the container, under Grafana's own leaf. Same shape as
# `swarm-bao-nats-callout-xkey` in ./swarm-nats.nix. The grant is
# `create` without `update`, so the first password stored is the one every
# later run reads, and a race between two runs leaves one value.
#
# Grafana does not start without the file (`$__file{}` on a missing path is
# a startup error), so a store that is unreachable on first boot keeps
# Grafana down rather than up with the upstream default.
systemd.services.swarm-bao-grafana-admin = {
description = "create Grafana's admin password in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-admin";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${ensureSecretDir}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's admin password file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# `bao kv get` exits 2 both for "nothing stored" and for a refusal, so
# a failed read always tries the create, and the second read decides.
# The value reaches `bao` on stdin, never in an argument.
if ! password="$(bao kv get -field=value ${lib.escapeShellArg adminPasswordStorePath} 2>"$err")"; then
minted="$(openssl rand -hex 32)"
if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg adminPasswordStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${adminPasswordStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! password="$(bao kv get -field=value ${lib.escapeShellArg adminPasswordStorePath} 2>"$err")"; then
echo "swarm-bao did not return ${adminPasswordStorePath}; Grafana does not start without it." >&2
cat "$err" >&2
exit 1
fi
fi
if [ -z "$password" ]; then
echo "swarm-bao returned an empty ${adminPasswordStorePath}; leaving the file as it is." >&2
exit 1
fi
# Owner and mode as for the OIDC secret above, for its reasons.
if secret_differs ${lib.escapeShellArg adminPasswordPath} "$password"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg adminPasswordPath} "$password"
fi
# Grafana's `preStart` applies the file to the database, so a file that
# changed after it started needs a restart.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg adminPasswordPath}
'';
};
# The metrics datasource's bearer: created in the store if it holds none,
# then copied into the container, under Grafana's own leaf. Same shape and
# grant as the admin password above. The store's `swarm-metrics-auth`
# reads the same path, so nothing is placed by hand on either host.
systemd.services.swarm-bao-grafana-metrics-bearer = {
description = "create Grafana's metrics store bearer in the swarm secret store if absent and fetch it";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
before = [ "container@${cfg.machine}.service" ];
wantedBy = [
"multi-user.target"
"container@${cfg.machine}.service"
];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.openssl
pkgs.systemd
];
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-bao-grafana-metrics-bearer";
TimeoutStartSec = 30;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
${atomicWriteSecret}
${refreshConsumer}
${ensureSecretDir}
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with Grafana's certificate; leaving Grafana's metrics bearer file as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# Read, create on a miss, read again: the admin password's sequence
# above, for its reasons.
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
minted="$(openssl rand -hex 32)"
if ! printf '%s' "$minted" | bao kv put -cas=0 ${lib.escapeShellArg metricsBearerStorePath} value=- >/dev/null 2>"$err"; then
echo "could not create ${metricsBearerStorePath}; reading it back in case another run created it." >&2
cat "$err" >&2
fi
if ! bearer="$(bao kv get -field=value ${lib.escapeShellArg metricsBearerStorePath} 2>"$err")"; then
echo "swarm-bao did not return ${metricsBearerStorePath}; Grafana does not start without it." >&2
cat "$err" >&2
exit 1
fi
fi
if [ -z "$bearer" ]; then
echo "swarm-bao returned an empty ${metricsBearerStorePath}; leaving the file as it is." >&2
exit 1
fi
# Owner and mode as for the OIDC secret above, for its reasons.
if secret_differs ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"; then
atomic_write_secret 0400 ${lib.escapeShellArg "${toString config.ids.uids.grafana}:0"} ${lib.escapeShellArg metricsBearerPath} "Bearer $bearer"
fi
# `$__file{}` in a provisioned datasource is read as Grafana starts.
refresh_consumer ${lib.escapeShellArg cfg.machine} grafana.service ${lib.escapeShellArg metricsBearerPath}
'';
};
# swarm-controller reads Grafana's alerts with a token of a Viewer service
# account, created here through Grafana's API as the built-in admin and
# handed over through the store.
#
# Re-run by the timer below. A stored token Grafana still accepts is kept;
# one it refuses with 401, or none at all, is replaced: every token of the
# account is deleted and one new one is minted and stored. The account's
# role is set back to Viewer on every run.
systemd.services.swarm-bao-grafana-controller-token = {
description = "give swarm-controller a Viewer token for Grafana's API through the swarm secret store";
# Ordered after the store where it runs here:
# ./glue-bao-readers-store-order.nix.
after = [
"container@${cfg.machine}.service"
"swarm-bao-grafana-admin.service"
];
wantedBy = [ "multi-user.target" ];
path = [
baoDeploy.package
pkgs.coreutils
pkgs.curl
pkgs.jq
];
# ./lib/store-retry.nix. A Grafana still starting refuses the
# connection, and that attempt fails into the same retry.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
# No `RemainAfterExit`: the timer below starts this again.
Type = "oneshot";
SyslogIdentifier = "swarm-bao-grafana-controller-token";
# Grafana's answers, the new token among them, pass through a temp file.
PrivateTmp = true;
# Each call below is bounded by curl's own timeouts; a run makes up to
# about ten of them.
TimeoutStartSec = "3min";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
BAO_CLIENT_CERT = baoDeploy.grafanaOidcClientCertFile;
BAO_CLIENT_KEY = baoDeploy.grafanaOidcClientKeyFile;
}
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
BAO_CACERT = baoDeploy.serverCaFile;
};
script = ''
set -euo pipefail
err="$(mktemp)"
body="$(mktemp)"
trap 'rm -f "$err" "$body"' EXIT
# One call to Grafana's API through its socket. Prints the HTTP
# status; the body lands in "$body". The credential reaches curl as
# a header file, never as an argument.
grafana() {
local auth="$1" method="$2" path="$3"
shift 3
curl --silent --show-error --connect-timeout 5 --max-time 15 \
--unix-socket ${lib.escapeShellArg socketPath} \
-H @<(printf 'Authorization: %s\n' "$auth") \
-H 'Content-Type: application/json' \
-X "$method" -o "$body" -w '%{http_code}' "$@" \
"http://localhost$path"
}
# Fails the attempt unless the last call answered one of the statuses
# given.
expect() {
local what="$1" status="$2"
shift 2
for ok in "$@"; do
[ "$status" = "$ok" ] && return 0
done
echo "Grafana answered $status to $what:" >&2
cat "$body" >&2
exit 1
}
if [ ! -s ${lib.escapeShellArg adminPasswordPath} ]; then
echo "no Grafana admin password at ${adminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
exit 1
fi
admin="Basic $(printf 'admin:%s' "$(< ${lib.escapeShellArg adminPasswordPath})" | base64 -w0)"
status="$(grafana "$admin" GET '/api/serviceaccounts/search?perpage=1000&query=${controllerAccountName}')"
expect "the service account search" "$status" 200
account="$(jq -c --arg n ${controllerAccountName} '[.serviceAccounts[] | select(.name == $n)][0] // empty' "$body")"
if [ -z "$account" ]; then
status="$(grafana "$admin" POST /api/serviceaccounts \
--data '{"name":"${controllerAccountName}","role":"Viewer","isDisabled":false}')"
expect "creating the ${controllerAccountName} service account" "$status" 200 201
account="$(jq -c . "$body")"
echo "created the ${controllerAccountName} service account as a Viewer"
fi
id="$(jq -r .id <<<"$account")"
if [ "$(jq -r '"\(.role) \(.isDisabled // false)"' <<<"$account")" != "Viewer false" ]; then
status="$(grafana "$admin" PATCH "/api/serviceaccounts/$id" \
--data '{"role":"Viewer","isDisabled":false}')"
expect "setting the ${controllerAccountName} service account to an enabled Viewer" "$status" 200
echo "set the ${controllerAccountName} service account back to an enabled Viewer"
fi
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with Grafana's certificate; leaving swarm-controller's Grafana token as it is." >&2
cat "$err" >&2
exit 1
fi
export BAO_TOKEN
# Checked against the call the controller makes with it.
if stored="$(bao kv get -field=value ${lib.escapeShellArg controllerTokenStorePath} 2>"$err")" && [ -n "$stored" ]; then
status="$(grafana "Bearer $stored" GET /api/prometheus/grafana/api/v1/alerts)"
case "$status" in
200) exit 0 ;;
401) echo "Grafana refuses the stored token for ${controllerAccountName}; replacing it." ;;
*) expect "reading alerts with the stored token" "$status" 200 ;;
esac
fi
status="$(grafana "$admin" GET "/api/serviceaccounts/$id/tokens")"
expect "listing the ${controllerAccountName} tokens" "$status" 200
for token_id in $(jq -r '.[].id' "$body"); do
status="$(grafana "$admin" DELETE "/api/serviceaccounts/$id/tokens/$token_id")"
expect "deleting token $token_id of ${controllerAccountName}" "$status" 200
done
status="$(grafana "$admin" POST "/api/serviceaccounts/$id/tokens" \
--data "{\"name\":\"${controllerAccountName}-$(date +%s)\"}")"
expect "minting a ${controllerAccountName} token" "$status" 200
key="$(jq -r .key "$body")"
: >"$body"
if ! printf '%s' "$key" | bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=- >/dev/null 2>"$err"; then
echo "could not store the new ${controllerAccountName} token at ${controllerTokenStorePath}; the next run replaces it." >&2
cat "$err" >&2
exit 1
fi
echo "stored a new ${controllerAccountName} token at ${controllerTokenStorePath}"
'';
};
systemd.timers.swarm-bao-grafana-controller-token = storeRefreshTimer "check swarm-controller's Grafana token and replace it if Grafana refuses it";
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# No `--link-journal=host`: the journal stays in the guest, where the
# container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
inherit privateNetwork;
# The socket directory, shared with the host so nginx can reach in.
#
# ⚠️ It has to be a bind mount rather than a path both sides happen to
# know: a container's `/run` is an nspawn tmpfs, so it is NOT visible
# from the host at `/var/lib/nixos-containers/<name>/run`. The host
# creates the directory (tmpfiles rule below, which also fixes
# ownership) and nspawn mounts it in — the same shape the per-agent
# `/run/hive-agent/<name>/web.sock` already uses.
bindMounts = {
${deployCfg.grafana.socketDir} = {
hostPath = deployCfg.grafana.socketDir;
isReadOnly = false;
};
# The three delivered secrets, read-only: the container reads them and
# never writes one.
${secretDir} = {
hostPath = secretDir;
isReadOnly = true;
};
}
// caTrust.bindMount;
config =
{ config, ... }:
{
imports = [
../container-modules/swarm-container.nix
../container-modules/swarm-journal-forwarder.nix
(import ../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
# The forwarder and its secret fetch dial the swarm collector,
# authelia and the store by name too.
dnsConsumers = [
"grafana.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
];
})
# Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store
# rather than adding to it — so it needs the system CAs and the
# store's root concatenated, not the root alone. Without it the
# browser half of the login succeeds and the server-to-server
# token exchange fails `x509: unknown authority`.
(caTrust.trustBundle {
inherit pkgs;
name = cfg.machine;
consumers = [
"grafana"
# Go too, for the swarm collector's and authelia's names.
"opentelemetry-collector"
];
})
];
services.hyperhive.swarmContainer = { inherit privateNetwork; };
# Authenticated as the forwarder's own OIDC client, with Grafana's
# store identity: one identity for the whole container. It is a
# system credential, so the fetch unit loads it.
services.hyperhive.swarmJournalForwarder = {
enable = true;
serviceName = "grafana";
endpoint = forwarderEndpoint;
clientId = forwarderClientId;
tokenUrl = "${autheliaUrl}/api/oidc/token";
telemetryPort = forwarderTelemetryPort;
store = {
package = baoDeploy.package;
addr = "https://${baoCfg.domain}:${toString baoCfg.port}";
secretPath = "secret/swarm/services/${forwarderClientId}/oidc/client";
clientCertFile = forwarderCredential "grafana-bao-cert";
clientKeyFile = forwarderCredential "grafana-bao-key";
caFile =
if storeFiles ? grafana-bao-server-ca then forwarderCredential "grafana-bao-server-ca" else null;
};
};
systemd.services.${forwarderUnit}.serviceConfig.LoadCredential = lib.attrNames storeFiles;
# Grafana's `secret_key` has **no default in nixpkgs** and an
# assertion refuses the build without one — which is how the first
# version of this module broke a deploy. It signs the datasource
# secrets in Grafana's own database.
#
# Generated in-container on first boot, like authelia's own keys and
# for the same reason: nothing outside this container ever reads it,
# which makes in-container generation right rather than merely
# easier. (The OIDC client secret is delivered host-side precisely
# because it has a second reader.)
#
# ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing.
# Rotating this key does not re-encrypt what it already encrypted, so
# a fresh key on every boot would leave Grafana unable to decrypt its
# own stored datasource secrets. Under /var/lib, never /run.
systemd.services.swarm-grafana-secret-key = {
description = "generate Grafana's datasource encryption key on first boot";
wantedBy = [ "grafana.service" ];
before = [ "grafana.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Same User/Group/StateDirectory as grafana itself, so systemd
# creates the directory owned by the account that has to read
# the file — no chown, no mode juggling.
User = "grafana";
Group = "grafana";
StateDirectory = "grafana-secret";
StateDirectoryMode = "0700";
SyslogIdentifier = "swarm-grafana-secret-key";
};
path = [
pkgs.openssl
pkgs.coreutils
];
script = ''
set -euo pipefail
${atomicWriteSecret}
key=${lib.escapeShellArg secretKeyPath}
if [ ! -s "$key" ]; then
value=$(openssl rand -hex 32)
atomic_write_secret 0600 "" "$key" "$value"
echo "generated Grafana's datasource encryption key"
fi
'';
};
# Upstream applies `security.admin_password` only when it creates the
# database, so a database created before that setting keeps the
# password it was created with. This resets it from the delivered
# file on every start, before the server listens.
#
# `grafana cli` must open the database the server opens, which the
# server's config.ini names. nixpkgs keeps that file private, so it
# is rendered again from the same settings with nixpkgs' formatter.
systemd.services.grafana.preStart =
let
grafana = config.services.grafana;
configFile =
(pkgs.formats.ini {
listToValue = lib.concatMapStringsSep " " (lib.generators.mkValueStringDefault { });
mkKeyValue = lib.generators.mkKeyValueDefault {
mkValueString = v: if v == null then "" else lib.generators.mkValueStringDefault { } v;
} "=";
}).generate
"config.ini"
grafana.settings;
in
lib.mkAfter ''
${lib.getExe' grafana.package "grafana"} cli --homepath ${grafana.dataDir} --config ${configFile} \
admin reset-admin-password --password-from-stdin < ${adminPasswordPath}
'';
# The group Grafana chgrps its socket to. Declared here because a
# container has its own user database — the host knowing gid 60 as
# `nginx` means nothing in here, only the number crosses.
#
# The name is local and arbitrary; the gid is the contract.
users.groups.gateway-nginx.gid = nginxGid;
users.users.grafana.extraGroups = [ "gateway-nginx" ];
services.grafana = {
enable = true;
package = deployCfg.grafana.package;
# Passed through unconditionally, empty default included.
# Upstream distinguishes `null` from `[ ]`, and both differences
# favour always handing it a list:
#
# - `null` points the plugin path at grafana's mutable
# `<dataDir>/plugins`; any list points it at a store path.
# Switching on the day someone adds their first plugin would
# bury that change inside an unrelated one.
# - upstream defaults its plugin update-check to
# `declarativePlugins == null`, so a list also stops the
# container phoning grafana.com. That is the no-runtime-egress
# property this option exists for — it should not arrive only
# once a plugin happens to be listed.
#
# - 🩸 it also sets upstream's `plugins.preinstall_disabled`
# (nixpkgs'
# grafana.nix defaults it to `declarativePlugins != null`,
# saying preinstall "causes issues with declarativePlugins").
# Preinstall is what fetches the Drilldown apps on first
# start, so claiming this directory silently removes them.
# That is why the option's default is not empty.
#
# Taking this directory over is not harmless on a hive with no
# plugins: that reasoning covers only plugins a PERSON installs,
# and upstream installs some itself.
declarativePlugins = deployCfg.grafana.plugins;
settings = {
server = {
# Already upstream's default (measured), but pinned rather
# than inherited: the gateway is the only intended client,
# and this being loopback is what keeps the UI from being
# published on whatever else the host is reachable on.
# A unix socket, not a port. `socket_gid` is nginx's, so the
# gateway can connect; `0660` keeps everyone else out.
#
# ⚠️ Grafana chowns the socket to `socket_gid`, and a
# non-root process may only chgrp to a group it is a MEMBER
# of — hence the group declared for `grafana` in this
# container's own user database below. Without that
# membership Grafana starts, fails the chown, and the socket
# is unreachable by nginx with nothing obviously wrong.
protocol = "socket";
socket = socketPath;
socket_gid = nginxGid;
socket_mode = "0660";
domain = cfg.domain;
# Grafana builds its own OAuth redirect from this. Left at
# upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it
# would name `http://<domain>:3000/`, which authelia has
# never heard of.
root_url = "https://${cfg.domain}/";
};
analytics = {
reporting_enabled = false;
check_for_updates = false;
};
# Stated rather than inherited. This is upstream's default, but
# a scrape target now depends on it, and a default that changes
# silently takes the target with it — the endpoint disappears,
# nginx keeps answering, and the scrape fails with a 404 far
# from anything that mentions Grafana.
metrics.enabled = true;
users.auto_assign_org_role = cfg.oidc.role;
# ⚠️ `$__file{}`, and required rather than optional: nixpkgs
# dropped the default and asserts on null, so without this the
# whole host fails to build. The unit below generates it.
security.secret_key = "$__file{${secretKeyPath}}";
# The password a newly created database gets; the `preStart`
# above gives an existing one the same. Written by
# `swarm-bao-grafana-admin` on the host.
security.admin_password = "$__file{${adminPasswordPath}}";
# No password box on the public login page: people log in through
# SSO. This hides the form only; the API still takes basic auth,
# which is why the admin password above is minted.
#
# Not conditional on anything, and the block below is not
# either, which is what keeps the two from ever disagreeing.
# Whether a password box sits on a public login page cannot
# depend on which host happens to run authelia.
auth.disable_login_form = true;
# Emitted in every deployment, because SSO is a requirement of
# running this service rather than a feature of some topologies:
# the assertions above refuse to build a Grafana that lacks an
# IdP to point at or the identity to fetch its secret with. A
# `lib.optionalAttrs` here would be the silent half of that pair,
# dropping the only login this container has whenever a value
# went null.
"auth.generic_oauth" = {
enabled = true;
name = "HyperHive";
client_id = cfg.oidc.clientId;
# ⚠️ `$__file{}`, never the secret itself — anything else
# here is world-readable in the nix store. Note the module's
# own leak assertion does NOT cover this key (it checks
# `database.password`, `security.admin_password` and
# datasource `secureJsonData`), so nothing but this comment
# stands between a literal and the store.
#
# `swarm-bao-grafana-oidc.service` above is what writes this
# path, on every host that runs Grafana.
client_secret = "$__file{${secretPath}}";
scopes = "openid profile email groups";
auth_url = "${autheliaUrl}/api/oidc/authorization";
token_url = "${autheliaUrl}/api/oidc/token";
api_url = "${autheliaUrl}/api/oidc/userinfo";
use_pkce = true;
};
};
provision.datasources.settings = {
apiVersion = 1;
datasources = [
{
name = "VictoriaMetrics";
type = "prometheus";
uid = datasourceUid;
url = deployCfg.grafana.datasourceUrl;
access = "proxy";
isDefault = true;
# The store's query routes admit this bearer or an operator's
# session, and nothing else.
jsonData.httpHeaderName1 = "Authorization";
secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}";
}
{
name = "VictoriaLogs";
# The plugin's own id, read from its `plugin.json` rather
# than guessed. A `type` Grafana does not recognise is
# provisioned without complaint and every query against it
# fails at use, far from anything naming this file.
type = "victoriametrics-logs-datasource";
uid = logsDatasourceUid;
url = deployCfg.grafana.logsDatasourceUrl;
access = "proxy";
# Exactly one datasource may claim this, and metrics has it:
# two defaults is a coin toss over which one an untyped panel
# gets.
isDefault = false;
# See `logLevelRules` above for why the level lives here and
# not in the pipeline.
jsonData = {
inherit logLevelRules;
httpHeaderName1 = "Authorization";
};
# The metrics datasource's bearer; the log store's vhost
# accepts it on `/select/logsql/`.
secureJsonData.httpHeaderValue1 = "$__file{${metricsBearerPath}}";
}
];
};
provision.dashboards.settings = {
apiVersion = 1;
providers = [
{
name = "hyperhive";
type = "file";
# Read-only in the UI: the store path is immutable, so an edit
# saved here would be silently discarded on the next rebuild.
# Better to refuse the edit than to lose it.
allowUiUpdates = false;
options.path = dashboardDir;
}
];
};
# Rules only. With no policy provisioned, Grafana routes every alert
# to its built-in `empty` receiver, which has no integrations: a
# firing rule shows under Alerting → Alert rules and is sent nowhere.
provision.alerting.rules.settings = {
apiVersion = 1;
groups = [
{
orgId = 1;
name = "hyperhive";
folder = "hyperhive";
interval = "1m";
rules = [
# The controller re-runs the forge objects pass every 5m
# (`RECONCILE_INTERVAL` in swarm-controller's
# forge/objects.rs) and warns once per failed write. A
# 10m window always spans the previous tick, so a pass
# failing on every tick keeps the count above zero
# continuously and fires after `for`. One failed pass
# keeps it above zero for only 10m, short of the 15m
# `for`, so it never fires; about three failed passes in
# a row do.
(logCountRule {
uid = "hh-forge-objects-write-failed";
title = "forge reconcile failing";
windowSeconds = 600;
expr = ''_time:10m _msg:"swarm forge objects: write failed; retrying next pass" | stats count() as failures'';
for = "15m";
summary = "swarm-controller has failed to write forge objects on consecutive reconcile passes";
})
# `hive` and `agent` are the resource attributes every
# agent container's log forwarder carries, so each agent
# is its own alert instance.
(logCountRule {
uid = "hh-swarm-term-publish-skipped";
title = "swarm terminal publish skipped";
windowSeconds = 900;
expr = ''_time:15m _msg:"swarm terminal: publish skipped" | stats by (hive, agent) count() as skipped'';
for = "5m";
summary = "{{ $labels.hive }}/{{ $labels.agent }} is dropping swarm terminal rows: the harness is not connected to the queue";
})
# Logged once per agent process: the connection is made
# once and never retried, so the agent stays unconnected
# until it restarts. The window is how long the rule
# stays firing after that one line.
(logCountRule {
uid = "hh-swarm-queue-connect-failed";
title = "swarm queue connect failed";
windowSeconds = 3600;
expr = ''_time:1h _msg:"swarm queue connect failed; this agent publishes nothing upward" | stats by (hive, agent) count() as failures'';
for = "0s";
summary = "{{ $labels.hive }}/{{ $labels.agent }} could not connect to the swarm queue and publishes nothing upward until it restarts";
})
(logCountRule {
uid = "hh-swarm-agent-state-publish-skipped";
title = "swarm agent state publish skipped";
windowSeconds = 900;
expr = ''_time:15m _msg:"swarm agent state: publish skipped" | stats by (hive, agent) count() as skipped'';
for = "5m";
summary = "{{ $labels.hive }}/{{ $labels.agent }} is dropping agent state updates: the harness is not connected to the queue";
})
# Logged on every connection attempt, so it repeats under
# the reconnect backoff for as long as the path is empty.
(logCountRule {
uid = "hh-swarm-queue-credential-missing";
title = "swarm queue credential missing";
windowSeconds = 900;
expr = ''_time:15m _msg:"nothing is stored at this agent's queue credential path; its queue connection retries under backoff" | stats by (hive, agent) count() as missing'';
for = "5m";
summary = "{{ $labels.hive }}/{{ $labels.agent }} finds nothing at its queue credential path and cannot connect to the queue";
})
# Logged once per failed turn, by `emit_turn_end` in
# hive-agent's turn.rs; the window is how long the rule
# stays firing after it. The exact `_msg` match and the
# unit keep out the subagent daemon's `subagent: turn
# failed` and the ACP backend's `ACP checkpoint turn
# failed`, which a phrase match would also count.
(logCountRule {
uid = "hh-agent-turn-failed";
title = "agent turn failed";
windowSeconds = 900;
expr = ''_time:15m _SYSTEMD_UNIT:="hive-agent.service" _msg:="turn failed" | stats by (hive, agent) count() as failures'';
for = "0s";
summary = "{{ $labels.hive }}/{{ $labels.agent }} had a claude turn fail; the reason is the `F_ERROR` field of the harness's `turn failed` log line";
})
# Same cadence as the forge rule: one pass every 5m
# (`RECONCILE_INTERVAL` in swarm-controller's
# agent_renewal.rs), one line per failed pass, so one
# failed pass never fires and about three in a row do.
(logCountRule {
uid = "hh-agent-credential-renewal-failed";
title = "agent credential renewal failing";
windowSeconds = 600;
expr = ''_time:10m _msg:"agent credential renewal: pass failed; retrying next tick" | stats count() as failures'';
for = "15m";
summary = "swarm-controller has failed the agent credential renewal pass on consecutive ticks";
})
# Logged once per refused deploy request, by
# `apply_rev` in hive-c0re's swarm_status.rs; the window is
# how long the rule stays firing after it. Host journal
# records carry the tracing fields as `F_<NAME>` and no
# `hive` attribute, so the machine is `_HOSTNAME`. The unit
# filter keeps out agent logs that quote the message.
(logCountRule {
uid = "hh-config-rev-refused";
title = "config rev refused";
windowSeconds = 900;
expr = ''_time:15m _SYSTEMD_UNIT:="hive-c0re.service" _msg:"swarm events: rev not applied; nothing queued" | stats by (_HOSTNAME, F_AGENT, F_REV) count() as refusals'';
for = "0s";
summary = "{{ $labels._HOSTNAME }} refused to deploy merged config commit {{ $labels.F_REV }} of agent {{ $labels.F_AGENT }}; the reason is the `F_ERROR` field of the hive-c0re log line";
})
# Logged once per detected crash, by `emit_crash_transitions`
# in hive-c0re's crash_watch.rs (10s poll interval). Replaces
# the old per-hive dashboard `agents_crashing` banner — see
# that function's module doc for why the banner went away
# in favour of this. Host journal records carry the
# tracing fields as `F_<NAME>` and no `hive` attribute,
# so the machine is `_HOSTNAME`
# (same shape as `hh-config-rev-refused` above). The unit
# filter keeps out an agent's own log line if one ever
# quotes the same message.
(logCountRule {
uid = "hh-container-crashed";
title = "container crashed";
windowSeconds = 900;
expr = ''_time:15m _SYSTEMD_UNIT:="hive-c0re.service" _msg:"container crash detected" | stats by (_HOSTNAME, F_AGENT) count() as crashes'';
for = "0s";
summary = "{{ $labels._HOSTNAME }} detected agent {{ $labels.F_AGENT }}'s container crash (not an operator-initiated stop)";
})
]
# One rule per declared hive rather than one grouped by
# `hive`, so a hive that sends nothing at all still has
# an instance to fire. `hive` is stamped by the swarm
# collector from the same `swarm.hives` key.
++ lib.mapAttrsToList (
name: _:
logAbsenceRule {
uid = "hh-logs-absent-${name}";
title = "no log records from ${name}";
windowSeconds = 600;
expr = ''_time:10m hive:="${name}" | stats count() as records'';
for = "5m";
summary = "hive ${name} has sent no log records to VictoriaLogs for at least 10m";
}
) hyperhiveCfg.swarm.hives;
}
];
};
};
};
};
};
}
View git blame Copy permalink