e01ecef18e
Second half of the metrics pair: a `swarm-grafana` container beside the VictoriaMetrics store, provisioned with it as the default datasource and fronted by the gateway on its own swarm-sibling name. Behind swarm SSO, per the operator's call on #3265. The authelia client and Grafana's callback URL both derive from `domain`, so the exact-match string authelia checks cannot drift from the one Grafana sends. The minted secret is delivered host-side (both container trees are only addressable there) and reaches Grafana as a `$__file{}` reference rather than a value, so it never enters the store. The login form is disabled whenever SSO is configured: Grafana ships an `admin`/`admin` account and this vhost is on the public gateway.
416 lines
17 KiB
Nix
416 lines
17 KiB
Nix
# The swarm's metrics UI: one Grafana for the whole swarm, in a
|
||
# `swarm-grafana` nixos-container next to the VictoriaMetrics store it reads.
|
||
#
|
||
# Two containers rather than one, on the operator's call: Grafana can be
|
||
# restarted, reconfigured or broken without taking the TSDB down with it.
|
||
# They are a pair, not a unit.
|
||
#
|
||
# Dashboards are provisioned from config, deliberately not deployed from git.
|
||
{
|
||
pkgs,
|
||
lib,
|
||
config,
|
||
...
|
||
}:
|
||
let
|
||
cfg = config.services.hyperhive.swarm.grafana;
|
||
hyperhiveCfg = config.services.hyperhive;
|
||
gatewayCfg = hyperhiveCfg.gateway;
|
||
tlsCfg = hyperhiveCfg.tls;
|
||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||
vmCfg = hyperhiveCfg.swarm.victoriametrics;
|
||
swarmDomain = hyperhiveCfg.swarm.domain;
|
||
|
||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||
useSelfSigned = caTrust.useSelfSigned;
|
||
|
||
# Total on a null swarm domain for the same reason every sibling module is:
|
||
# the required-domain assertion in hive-network.nix should be what an
|
||
# operator sees, not a coercion error from here.
|
||
domainBase = if swarmDomain == null then "invalid" else swarmDomain;
|
||
|
||
# The all-local case: this host runs BOTH Grafana and the swarm's authelia,
|
||
# so the minted secret can be moved without an operator. Same split the
|
||
# forge and matrix modules document.
|
||
ssoLocal = cfg.enable && autheliaCfg.enable;
|
||
autheliaUrl = toString autheliaCfg.url;
|
||
|
||
# Where the plaintext lands inside the container. Under /var/lib rather
|
||
# than /run: Grafana may start before the delivery unit on a later boot,
|
||
# and a secret that evaporates on reboot turns a working login into an
|
||
# intermittent one.
|
||
secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret";
|
||
|
||
# Format-locked by Grafana: the generic OAuth callback is always
|
||
# `<root_url>/login/generic_oauth`. Declared once here and read by both
|
||
# the authelia client and Grafana itself.
|
||
redirectUri = "https://${cfg.domain}/login/generic_oauth";
|
||
|
||
grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt";
|
||
in
|
||
{
|
||
options.services.hyperhive.swarm.grafana = {
|
||
enable = lib.mkOption {
|
||
type = lib.types.bool;
|
||
default = false;
|
||
description = ''
|
||
Run the swarm's metrics UI on this host. Off by default and not
|
||
derived from {option}`services.hyperhive.enable`: a swarm has one
|
||
Grafana, so enabling it is a decision about swarm topology rather
|
||
than about whether hyperhive is installed.
|
||
'';
|
||
};
|
||
|
||
package = lib.mkOption {
|
||
type = lib.types.package;
|
||
default = pkgs.grafana;
|
||
defaultText = lib.literalExpression "pkgs.grafana";
|
||
description = "Grafana package to run.";
|
||
};
|
||
|
||
machine = lib.mkOption {
|
||
type = lib.types.str;
|
||
readOnly = true;
|
||
default = "swarm-grafana";
|
||
description = ''
|
||
Container name. Read-only: the name appears in host paths and in
|
||
`machinectl`, so it is a fact other modules may read rather than a
|
||
knob.
|
||
'';
|
||
};
|
||
|
||
domain = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "grafana.${domainBase}";
|
||
defaultText = lib.literalExpression ''"grafana.''${services.hyperhive.swarm.domain}"'';
|
||
description = ''
|
||
Name the gateway serves this on. A sibling of the swarm's other
|
||
service names, so the swarm-services sub-CA can issue for it — see
|
||
`hive-tls.nix` for why a service name being a sibling rather than a
|
||
child decides which CA may sign it.
|
||
|
||
⚠️ Changing this changes the OAuth redirect URI, which authelia
|
||
matches exactly. Both sides move together because both derive from
|
||
this option; an operator who pins one by hand breaks the login.
|
||
'';
|
||
};
|
||
|
||
port = lib.mkOption {
|
||
type = lib.types.port;
|
||
default = 3000;
|
||
description = ''
|
||
Port Grafana listens on, bound to loopback only. Upstream's own
|
||
default, kept so an operator reading Grafana documentation finds
|
||
what they expect.
|
||
'';
|
||
};
|
||
|
||
datasourceUrl = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "http://127.0.0.1:${toString vmCfg.port}";
|
||
defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"'';
|
||
description = ''
|
||
Where the provisioned datasource points. Defaults to the metrics
|
||
store on this host, which is the only place it can be: that store
|
||
binds loopback, so a Grafana somewhere else could not reach it
|
||
anyway. Set explicitly if a deployment fronts VictoriaMetrics with
|
||
something that does listen wider.
|
||
'';
|
||
};
|
||
|
||
oidc = {
|
||
clientId = lib.mkOption {
|
||
type = lib.types.str;
|
||
default = "swarm-grafana";
|
||
description = ''
|
||
The authelia OIDC client id. Names the application rather than
|
||
the protocol, per the convention in
|
||
{option}`services.hyperhive.swarm.authelia.oidc.clients`.
|
||
'';
|
||
};
|
||
|
||
role = lib.mkOption {
|
||
type = lib.types.enum [
|
||
"Viewer"
|
||
"Editor"
|
||
"Admin"
|
||
];
|
||
default = "Admin";
|
||
example = "Editor";
|
||
description = ''
|
||
Grafana org role every SSO user is assigned.
|
||
|
||
`Admin` by default, and that is a considered default rather than
|
||
a permissive one: the login form is disabled whenever SSO is
|
||
configured, so this is the *only* way anyone reaches Grafana —
|
||
a `Viewer` default would produce a swarm nobody can administer.
|
||
Passing authelia already means being an operator of this swarm;
|
||
its user store is the small, `swarmctl`-managed one.
|
||
|
||
Lower it if a swarm ever grows read-only operators, which is a
|
||
one-line change here.
|
||
'';
|
||
};
|
||
};
|
||
};
|
||
|
||
config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) {
|
||
# The gateway name and the quick-link, both inside `cfg.enable` — that
|
||
# guard is the load-bearing part. Every hive in a swarm may know this UI
|
||
# exists, but only the host that RUNS it may claim the name; a client
|
||
# hive declaring the vhost would answer for a service it does not have.
|
||
services.hyperhive.gateway.localNames = [ cfg.domain ];
|
||
|
||
services.hyperhive.swarm.controller.links = [
|
||
{
|
||
label = "Grafana";
|
||
icon = "📊";
|
||
url = "https://${cfg.domain}/";
|
||
}
|
||
];
|
||
|
||
# One declaration, two readers. Grafana's callback URL is format-locked
|
||
# to its own root URL; making the operator restate it in authelia's
|
||
# client list would be a second source of truth for a string whose
|
||
# mismatch is a silently rejected login.
|
||
#
|
||
# `kind` is left at its `interactive` default: a person logs in here.
|
||
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [
|
||
{
|
||
id = cfg.oidc.clientId;
|
||
description = "HyperHive swarm metrics";
|
||
redirectUris = [ redirectUri ];
|
||
}
|
||
];
|
||
|
||
assertions = [
|
||
{
|
||
# Grafana reaches the token endpoint server-to-server, so a null URL
|
||
# would become a request to `null/api/oidc/token` — a DNS failure
|
||
# several layers from its cause. Only reachable by enabling authelia
|
||
# and clearing its `url`, which is why it is an assertion and not a
|
||
# fallback.
|
||
assertion = !ssoLocal || autheliaCfg.url != null;
|
||
message = ''
|
||
services.hyperhive.swarm.grafana requires
|
||
services.hyperhive.swarm.authelia.url when authelia is enabled.
|
||
|
||
Grafana exchanges its authorization code at
|
||
`''${url}/api/oidc/token` from inside its container. With the URL
|
||
null there is no endpoint to name.
|
||
'';
|
||
}
|
||
];
|
||
|
||
# Websockets: Grafana Live streams panel updates over one, and without
|
||
# the upgrade headers dashboards load and then never refresh — which
|
||
# reads as stale data rather than as a proxy fault.
|
||
services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // {
|
||
listen = gatewayCfg.lib.listen;
|
||
extraConfig = gatewayCfg.lib.securityHeaders;
|
||
locations."/" = {
|
||
proxyPass = "http://127.0.0.1:${toString cfg.port}/";
|
||
proxyWebsockets = true;
|
||
extraConfig = ''
|
||
# Grafana builds its OAuth redirect from the ORIGINAL request.
|
||
# Without these every request looks like it arrived at 127.0.0.1
|
||
# over plain http, and the redirect sent to authelia names a
|
||
# host the browser cannot reach.
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
proxy_set_header X-Forwarded-Host $host;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
'';
|
||
};
|
||
};
|
||
|
||
# Order the container after the host CA service so the bind source below
|
||
# exists before nspawn sets the mount up.
|
||
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
|
||
|
||
# The secret delivery. It runs on the HOST because that is the only place
|
||
# both container trees are addressable: they share this host's network
|
||
# namespace, which makes them feel co-located, but their filesystem roots
|
||
# are separate — Grafana cannot open a path inside authelia's tree
|
||
# however local the port looks.
|
||
#
|
||
# ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container
|
||
# refuses to start when a bind source is missing, and this secret does
|
||
# not exist until authelia's first boot has minted it — so binding it
|
||
# would make Grafana wait on a file that waits on a container that starts
|
||
# after it.
|
||
systemd.services.swarm-grafana-oidc-secret = lib.mkIf ssoLocal {
|
||
description = "deliver Grafana's OIDC client secret from authelia";
|
||
after = [ "container@${autheliaCfg.machine}.service" ];
|
||
requires = [ "container@${autheliaCfg.machine}.service" ];
|
||
before = [ "container@${cfg.machine}.service" ];
|
||
wantedBy = [ "container@${cfg.machine}.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-grafana-oidc-secret";
|
||
# ⚠️ Longer than the wait below, and that is the whole point:
|
||
# `DefaultTimeoutStartSec` is 90s, so a 120s bounded wait is killed
|
||
# by systemd at 90 — before it can emit the error naming the file it
|
||
# waited for. The timeout has to outlive the thing it is timing.
|
||
TimeoutStartSec = "180s";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
|
||
src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.oidc.clientId}.secret"}
|
||
dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"}
|
||
|
||
# authelia's container is up, but its first-boot generator may still
|
||
# be minting. Bounded wait, then fail: a silent skip here produces a
|
||
# Grafana whose only login path dead-ends.
|
||
deadline=$(( SECONDS + 120 ))
|
||
while [ ! -s "$src" ]; do
|
||
if [ "$SECONDS" -ge "$deadline" ]; then
|
||
echo "authelia has not minted $src after 120s" >&2
|
||
exit 1
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
# Owned by Grafana's own uid, unlike the matrix sibling which lands
|
||
# root-owned: tuwunel's secret is read by `LoadCredential` as root
|
||
# before the sandbox exists, whereas Grafana expands `$__file{}`
|
||
# itself, as itself, while parsing its config. These containers set
|
||
# no `privateUsers`, so the host uid is the container uid, and both
|
||
# sides take it from the same static NixOS id.
|
||
#
|
||
# Group is root, not grafana, and that is forced rather than chosen:
|
||
# `ids.uids.grafana` is a static id but there is no `ids.gids.grafana`
|
||
# — the group's gid is allocated at activation inside the container,
|
||
# so the host cannot know it at eval time. Harmless here because 0400
|
||
# grants the group nothing; if this mode ever widens, the gid has to
|
||
# be discovered at runtime rather than assumed.
|
||
install -D -m 0400 -o ${toString config.ids.uids.grafana} -g 0 "$src" "$dst"
|
||
'';
|
||
};
|
||
|
||
containers.${cfg.machine} = {
|
||
autoStart = true;
|
||
ephemeral = false;
|
||
# Shared host netns, like every sibling swarm container: the gateway
|
||
# reaches this at 127.0.0.1:<port>.
|
||
privateNetwork = false;
|
||
|
||
bindMounts = { } // caTrust.bindMount;
|
||
|
||
config =
|
||
{ ... }:
|
||
{
|
||
system.stateVersion = "26.05";
|
||
|
||
# This container shares the host netns, so its own firewall.service
|
||
# would rewrite the HOST ruleset at every boot. The host firewall
|
||
# owns all filtering.
|
||
networking.firewall.enable = false;
|
||
# Keep the host-copied /etc/resolv.conf intact — resolvconf's
|
||
# host-tracking would regenerate it to an empty file, since the
|
||
# host's copy doesn't cross the boundary after start.
|
||
networking.resolvconf.enable = lib.mkForce false;
|
||
|
||
# Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE`
|
||
# *replaces* the default bundle rather than adding to it — so
|
||
# concatenate the system CAs with the bind-mounted hive CA instead
|
||
# of pointing at the CA alone, which would lose every public
|
||
# anchor. /run is tmpfs, so this is rebuilt from the current CA
|
||
# each boot rather than going stale.
|
||
#
|
||
# Without it the browser half of the login succeeds and the
|
||
# server-to-server token exchange fails with an x509 "unknown
|
||
# authority" — the same shape of failure the swarm queue hit.
|
||
systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned {
|
||
description = "assemble Grafana TLS trust bundle (system CAs + hive CA)";
|
||
wantedBy = [ "grafana.service" ];
|
||
before = [ "grafana.service" ];
|
||
serviceConfig = {
|
||
Type = "oneshot";
|
||
RemainAfterExit = true;
|
||
SyslogIdentifier = "swarm-grafana-ca-bundle";
|
||
};
|
||
path = [ pkgs.coreutils ];
|
||
script = ''
|
||
set -euo pipefail
|
||
install -d -m 0755 /run/swarm-grafana-ca
|
||
cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \
|
||
> ${grafanaCaBundle}
|
||
chmod 0644 ${grafanaCaBundle}
|
||
'';
|
||
};
|
||
|
||
systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle;
|
||
|
||
services.grafana = {
|
||
enable = true;
|
||
package = cfg.package;
|
||
|
||
settings = {
|
||
server = {
|
||
# Already upstream's default (measured), but pinned rather
|
||
# than inherited: the gateway is the only intended client,
|
||
# and this being loopback is what keeps the UI from being
|
||
# published on whatever else the host is reachable on.
|
||
http_addr = "127.0.0.1";
|
||
http_port = cfg.port;
|
||
domain = cfg.domain;
|
||
# Grafana builds its own OAuth redirect from this. Left at
|
||
# upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it
|
||
# would name `http://<domain>:3000/`, which authelia has
|
||
# never heard of.
|
||
root_url = "https://${cfg.domain}/";
|
||
};
|
||
|
||
analytics = {
|
||
reporting_enabled = false;
|
||
check_for_updates = false;
|
||
};
|
||
|
||
users.auto_assign_org_role = cfg.oidc.role;
|
||
|
||
# No local password path at all when SSO is configured. This
|
||
# is not tidiness: Grafana ships an `admin`/`admin` account,
|
||
# and this vhost is on the public gateway.
|
||
auth.disable_login_form = ssoLocal;
|
||
}
|
||
// lib.optionalAttrs ssoLocal {
|
||
"auth.generic_oauth" = {
|
||
enabled = true;
|
||
name = "HyperHive";
|
||
client_id = cfg.oidc.clientId;
|
||
# ⚠️ `$__file{}`, never the secret itself — anything else
|
||
# here is world-readable in the nix store. Note the module's
|
||
# own leak assertion does NOT cover this key (it checks
|
||
# `database.password`, `security.admin_password` and
|
||
# datasource `secureJsonData`), so nothing but this comment
|
||
# stands between a literal and the store.
|
||
client_secret = "$__file{${secretPath}}";
|
||
scopes = "openid profile email groups";
|
||
auth_url = "${autheliaUrl}/api/oidc/authorization";
|
||
token_url = "${autheliaUrl}/api/oidc/token";
|
||
api_url = "${autheliaUrl}/api/oidc/userinfo";
|
||
use_pkce = true;
|
||
};
|
||
};
|
||
|
||
provision.datasources.settings = {
|
||
apiVersion = 1;
|
||
datasources = [
|
||
{
|
||
name = "VictoriaMetrics";
|
||
type = "prometheus";
|
||
uid = "swarm-victoriametrics";
|
||
url = cfg.datasourceUrl;
|
||
access = "proxy";
|
||
isDefault = true;
|
||
}
|
||
];
|
||
};
|
||
};
|
||
};
|
||
};
|
||
};
|
||
}
|