vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(#3265): swarm metrics UI as a Grafana container

Browse source 
Second half of the metrics pair: a `swarm-grafana` container beside the
VictoriaMetrics store, provisioned with it as the default datasource and
fronted by the gateway on its own swarm-sibling name.

Behind swarm SSO, per the operator's call on #3265. The authelia client
and Grafana's callback URL both derive from `domain`, so the exact-match
string authelia checks cannot drift from the one Grafana sends. The
minted secret is delivered host-side (both container trees are only
addressable there) and reaches Grafana as a `$__file{}` reference rather
than a value, so it never enters the store.

The login form is disabled whenever SSO is configured: Grafana ships an
`admin`/`admin` account and this vhost is on the public gateway.
This commit is contained in:
atlas 2026-08-16 21:38:24 +02:00 • committed by mara
commit e01ecef18e
2 changed files with 417 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
nix/host-modules/default.nix
View file

@ -26,6 +26,7 @@
./swarm-ca.nix
./swarm-nats.nix
./swarm-controller.nix
./swarm-grafana.nix
./swarm-snapshot-store.nix
./swarm-ui.nix
./swarm-victoriametrics.nix

416
nix/host-modules/swarm-grafana.nix Normal file
View file

@ -0,0 +1,416 @@
# The swarm's metrics UI: one Grafana for the whole swarm, in a
# `swarm-grafana` nixos-container next to the VictoriaMetrics store it reads.
#
# Two containers rather than one, on the operator's call: Grafana can be
# restarted, reconfigured or broken without taking the TSDB down with it.
# They are a pair, not a unit.
#
# Dashboards are provisioned from config, deliberately not deployed from git.
{
pkgs,
lib,
config,
...
}:
let
cfg = config.services.hyperhive.swarm.grafana;
hyperhiveCfg = config.services.hyperhive;
gatewayCfg = hyperhiveCfg.gateway;
tlsCfg = hyperhiveCfg.tls;
autheliaCfg = hyperhiveCfg.swarm.authelia;
vmCfg = hyperhiveCfg.swarm.victoriametrics;
swarmDomain = hyperhiveCfg.swarm.domain;
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
useSelfSigned = caTrust.useSelfSigned;
# Total on a null swarm domain for the same reason every sibling module is:
# the required-domain assertion in hive-network.nix should be what an
# operator sees, not a coercion error from here.
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
# The all-local case: this host runs BOTH Grafana and the swarm's authelia,
# so the minted secret can be moved without an operator. Same split the
# forge and matrix modules document.
ssoLocal = cfg.enable && autheliaCfg.enable;
autheliaUrl = toString autheliaCfg.url;
# Where the plaintext lands inside the container. Under /var/lib rather
# than /run: Grafana may start before the delivery unit on a later boot,
# and a secret that evaporates on reboot turns a working login into an
# intermittent one.
secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret";
# Format-locked by Grafana: the generic OAuth callback is always
# `<root_url>/login/generic_oauth`. Declared once here and read by both
# the authelia client and Grafana itself.
redirectUri = "https://${cfg.domain}/login/generic_oauth";
grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt";
in
{
options.services.hyperhive.swarm.grafana = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
Run the swarm's metrics UI on this host. Off by default and not
derived from {option}`services.hyperhive.enable`: a swarm has one
Grafana, so enabling it is a decision about swarm topology rather
than about whether hyperhive is installed.
'';
};
package = lib.mkOption {
type = lib.types.package;
default = pkgs.grafana;
defaultText = lib.literalExpression "pkgs.grafana";
description = "Grafana package to run.";
};
machine = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = "swarm-grafana";
description = ''
Container name. Read-only: the name appears in host paths and in
`machinectl`, so it is a fact other modules may read rather than a
knob.
'';
};
domain = lib.mkOption {
type = lib.types.str;
default = "grafana.${domainBase}";
defaultText = lib.literalExpression ''"grafana.''${services.hyperhive.swarm.domain}"'';
description = ''
Name the gateway serves this on. A sibling of the swarm's other
service names, so the swarm-services sub-CA can issue for it — see
`hive-tls.nix` for why a service name being a sibling rather than a
child decides which CA may sign it.
⚠️ Changing this changes the OAuth redirect URI, which authelia
matches exactly. Both sides move together because both derive from
this option; an operator who pins one by hand breaks the login.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 3000;
description = ''
Port Grafana listens on, bound to loopback only. Upstream's own
default, kept so an operator reading Grafana documentation finds
what they expect.
'';
};
datasourceUrl = lib.mkOption {
type = lib.types.str;
default = "http://127.0.0.1:${toString vmCfg.port}";
defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"'';
description = ''
Where the provisioned datasource points. Defaults to the metrics
store on this host, which is the only place it can be: that store
binds loopback, so a Grafana somewhere else could not reach it
anyway. Set explicitly if a deployment fronts VictoriaMetrics with
something that does listen wider.
'';
};
oidc = {
clientId = lib.mkOption {
type = lib.types.str;
default = "swarm-grafana";
description = ''
The authelia OIDC client id. Names the application rather than
the protocol, per the convention in
{option}`services.hyperhive.swarm.authelia.oidc.clients`.
'';
};
role = lib.mkOption {
type = lib.types.enum [
"Viewer"
"Editor"
"Admin"
];
default = "Admin";
example = "Editor";
description = ''
Grafana org role every SSO user is assigned.
`Admin` by default, and that is a considered default rather than
a permissive one: the login form is disabled whenever SSO is
configured, so this is the *only* way anyone reaches Grafana —
a `Viewer` default would produce a swarm nobody can administer.
Passing authelia already means being an operator of this swarm;
its user store is the small, `swarmctl`-managed one.
Lower it if a swarm ever grows read-only operators, which is a
one-line change here.
'';
};
};
};
config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) {
# The gateway name and the quick-link, both inside `cfg.enable` — that
# guard is the load-bearing part. Every hive in a swarm may know this UI
# exists, but only the host that RUNS it may claim the name; a client
# hive declaring the vhost would answer for a service it does not have.
services.hyperhive.gateway.localNames = [ cfg.domain ];
services.hyperhive.swarm.controller.links = [
{
label = "Grafana";
icon = "📊";
url = "https://${cfg.domain}/";
}
];
# One declaration, two readers. Grafana's callback URL is format-locked
# to its own root URL; making the operator restate it in authelia's
# client list would be a second source of truth for a string whose
# mismatch is a silently rejected login.
#
# `kind` is left at its `interactive` default: a person logs in here.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [
{
id = cfg.oidc.clientId;
description = "HyperHive swarm metrics";
redirectUris = [ redirectUri ];
}
];
assertions = [
{
# Grafana reaches the token endpoint server-to-server, so a null URL
# would become a request to `null/api/oidc/token` — a DNS failure
# several layers from its cause. Only reachable by enabling authelia
# and clearing its `url`, which is why it is an assertion and not a
# fallback.
assertion = !ssoLocal || autheliaCfg.url != null;
message = ''
services.hyperhive.swarm.grafana requires
services.hyperhive.swarm.authelia.url when authelia is enabled.
Grafana exchanges its authorization code at
`''${url}/api/oidc/token` from inside its container. With the URL
null there is no endpoint to name.
'';
}
];
# Websockets: Grafana Live streams panel updates over one, and without
# the upgrade headers dashboards load and then never refresh — which
# reads as stale data rather than as a proxy fault.
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
listen = gatewayCfg.lib.listen;
extraConfig = gatewayCfg.lib.securityHeaders;
locations."/" = {
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
proxyWebsockets = true;
extraConfig = ''
# Grafana builds its OAuth redirect from the ORIGINAL request.
# Without these every request looks like it arrived at 127.0.0.1
# over plain http, and the redirect sent to authelia names a
# host the browser cannot reach.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
'';
};
};
# Order the container after the host CA service so the bind source below
# exists before nspawn sets the mount up.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
# The secret delivery. It runs on the HOST because that is the only place
# both container trees are addressable: they share this host's network
# namespace, which makes them feel co-located, but their filesystem roots
# are separate — Grafana cannot open a path inside authelia's tree
# however local the port looks.
#
# ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
# refuses to start when a bind source is missing, and this secret does
# not exist until authelia's first boot has minted it — so binding it
# would make Grafana wait on a file that waits on a container that starts
# after it.
systemd.services.swarm-grafana-oidc-secret = lib.mkIf ssoLocal {
description = "deliver Grafana's OIDC client secret from authelia";
after = [ "container@${autheliaCfg.machine}.service" ];
requires = [ "container@${autheliaCfg.machine}.service" ];
before = [ "container@${cfg.machine}.service" ];
wantedBy = [ "container@${cfg.machine}.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-grafana-oidc-secret";
# ⚠️ Longer than the wait below, and that is the whole point:
# `DefaultTimeoutStartSec` is 90s, so a 120s bounded wait is killed
# by systemd at 90 — before it can emit the error naming the file it
# waited for. The timeout has to outlive the thing it is timing.
TimeoutStartSec = "180s";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.oidc.clientId}.secret"}
dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"}
# authelia's container is up, but its first-boot generator may still
# be minting. Bounded wait, then fail: a silent skip here produces a
# Grafana whose only login path dead-ends.
deadline=$(( SECONDS + 120 ))
while [ ! -s "$src" ]; do
if [ "$SECONDS" -ge "$deadline" ]; then
echo "authelia has not minted $src after 120s" >&2
exit 1
fi
sleep 2
done
# Owned by Grafana's own uid, unlike the matrix sibling which lands
# root-owned: tuwunel's secret is read by `LoadCredential` as root
# before the sandbox exists, whereas Grafana expands `$__file{}`
# itself, as itself, while parsing its config. These containers set
# no `privateUsers`, so the host uid is the container uid, and both
# sides take it from the same static NixOS id.
#
# Group is root, not grafana, and that is forced rather than chosen:
# `ids.uids.grafana` is a static id but there is no `ids.gids.grafana`
# — the group's gid is allocated at activation inside the container,
# so the host cannot know it at eval time. Harmless here because 0400
# grants the group nothing; if this mode ever widens, the gid has to
# be discovered at runtime rather than assumed.
install -D -m 0400 -o ${toString config.ids.uids.grafana} -g 0 "$src" "$dst"
'';
};
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;
# Shared host netns, like every sibling swarm container: the gateway
# reaches this at 127.0.0.1:<port>.
privateNetwork = false;
bindMounts = { } // caTrust.bindMount;
config =
{ ... }:
{
system.stateVersion = "26.05";
# This container shares the host netns, so its own firewall.service
# would rewrite the HOST ruleset at every boot. The host firewall
# owns all filtering.
networking.firewall.enable = false;
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
# host-tracking would regenerate it to an empty file, since the
# host's copy doesn't cross the boundary after start.
networking.resolvconf.enable = lib.mkForce false;
# Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE`
# *replaces* the default bundle rather than adding to it — so
# concatenate the system CAs with the bind-mounted hive CA instead
# of pointing at the CA alone, which would lose every public
# anchor. /run is tmpfs, so this is rebuilt from the current CA
# each boot rather than going stale.
#
# Without it the browser half of the login succeeds and the
# server-to-server token exchange fails with an x509 "unknown
# authority" — the same shape of failure the swarm queue hit.
systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned {
description = "assemble Grafana TLS trust bundle (system CAs + hive CA)";
wantedBy = [ "grafana.service" ];
before = [ "grafana.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "swarm-grafana-ca-bundle";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
install -d -m 0755 /run/swarm-grafana-ca
cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \
> ${grafanaCaBundle}
chmod 0644 ${grafanaCaBundle}
'';
};
systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle;
services.grafana = {
enable = true;
package = cfg.package;
settings = {
server = {
# Already upstream's default (measured), but pinned rather
# than inherited: the gateway is the only intended client,
# and this being loopback is what keeps the UI from being
# published on whatever else the host is reachable on.
http_addr = "127.0.0.1";
http_port = cfg.port;
domain = cfg.domain;
# Grafana builds its own OAuth redirect from this. Left at
# upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it
# would name `http://<domain>:3000/`, which authelia has
# never heard of.
root_url = "https://${cfg.domain}/";
};
analytics = {
reporting_enabled = false;
check_for_updates = false;
};
users.auto_assign_org_role = cfg.oidc.role;
# No local password path at all when SSO is configured. This
# is not tidiness: Grafana ships an `admin`/`admin` account,
# and this vhost is on the public gateway.
auth.disable_login_form = ssoLocal;
}
// lib.optionalAttrs ssoLocal {
"auth.generic_oauth" = {
enabled = true;
name = "HyperHive";
client_id = cfg.oidc.clientId;
# ⚠️ `$__file{}`, never the secret itself — anything else
# here is world-readable in the nix store. Note the module's
# own leak assertion does NOT cover this key (it checks
# `database.password`, `security.admin_password` and
# datasource `secureJsonData`), so nothing but this comment
# stands between a literal and the store.
client_secret = "$__file{${secretPath}}";
scopes = "openid profile email groups";
auth_url = "${autheliaUrl}/api/oidc/authorization";
token_url = "${autheliaUrl}/api/oidc/token";
api_url = "${autheliaUrl}/api/oidc/userinfo";
use_pkce = true;
};
};
provision.datasources.settings = {
apiVersion = 1;
datasources = [
{
name = "VictoriaMetrics";
type = "prometheus";
uid = "swarm-victoriametrics";
url = cfg.datasourceUrl;
access = "proxy";
isDefault = true;
}
];
};
};
};
};
};
}