From e01ecef18e12c4294ad56672b5e327c7cce0c241 Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 16 Aug 2026 21:38:24 +0200 Subject: [PATCH] feat(#3265): swarm metrics UI as a Grafana container Second half of the metrics pair: a `swarm-grafana` container beside the VictoriaMetrics store, provisioned with it as the default datasource and fronted by the gateway on its own swarm-sibling name. Behind swarm SSO, per the operator's call on #3265. The authelia client and Grafana's callback URL both derive from `domain`, so the exact-match string authelia checks cannot drift from the one Grafana sends. The minted secret is delivered host-side (both container trees are only addressable there) and reaches Grafana as a `$__file{}` reference rather than a value, so it never enters the store. The login form is disabled whenever SSO is configured: Grafana ships an `admin`/`admin` account and this vhost is on the public gateway. --- nix/host-modules/default.nix | 1 + nix/host-modules/swarm-grafana.nix | 416 +++++++++++++++++++++++++++++ 2 files changed, 417 insertions(+) create mode 100644 nix/host-modules/swarm-grafana.nix diff --git a/nix/host-modules/default.nix b/nix/host-modules/default.nix index 9d33467f..d505a6a2 100644 --- a/nix/host-modules/default.nix +++ b/nix/host-modules/default.nix @@ -26,6 +26,7 @@ ./swarm-ca.nix ./swarm-nats.nix ./swarm-controller.nix + ./swarm-grafana.nix ./swarm-snapshot-store.nix ./swarm-ui.nix ./swarm-victoriametrics.nix diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix new file mode 100644 index 00000000..111bdc03 --- /dev/null +++ b/nix/host-modules/swarm-grafana.nix @@ -0,0 +1,416 @@ +# The swarm's metrics UI: one Grafana for the whole swarm, in a +# `swarm-grafana` nixos-container next to the VictoriaMetrics store it reads. +# +# Two containers rather than one, on the operator's call: Grafana can be +# restarted, reconfigured or broken without taking the TSDB down with it. +# They are a pair, not a unit. +# +# Dashboards are provisioned from config, deliberately not deployed from git. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.swarm.grafana; + hyperhiveCfg = config.services.hyperhive; + gatewayCfg = hyperhiveCfg.gateway; + tlsCfg = hyperhiveCfg.tls; + autheliaCfg = hyperhiveCfg.swarm.authelia; + vmCfg = hyperhiveCfg.swarm.victoriametrics; + swarmDomain = hyperhiveCfg.swarm.domain; + + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; + useSelfSigned = caTrust.useSelfSigned; + + # Total on a null swarm domain for the same reason every sibling module is: + # the required-domain assertion in hive-network.nix should be what an + # operator sees, not a coercion error from here. + domainBase = if swarmDomain == null then "invalid" else swarmDomain; + + # The all-local case: this host runs BOTH Grafana and the swarm's authelia, + # so the minted secret can be moved without an operator. Same split the + # forge and matrix modules document. + ssoLocal = cfg.enable && autheliaCfg.enable; + autheliaUrl = toString autheliaCfg.url; + + # Where the plaintext lands inside the container. Under /var/lib rather + # than /run: Grafana may start before the delivery unit on a later boot, + # and a secret that evaporates on reboot turns a working login into an + # intermittent one. + secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"; + + # Format-locked by Grafana: the generic OAuth callback is always + # `/login/generic_oauth`. Declared once here and read by both + # the authelia client and Grafana itself. + redirectUri = "https://${cfg.domain}/login/generic_oauth"; + + grafanaCaBundle = "/run/swarm-grafana-ca/ca-bundle.crt"; +in +{ + options.services.hyperhive.swarm.grafana = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Run the swarm's metrics UI on this host. Off by default and not + derived from {option}`services.hyperhive.enable`: a swarm has one + Grafana, so enabling it is a decision about swarm topology rather + than about whether hyperhive is installed. + ''; + }; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.grafana; + defaultText = lib.literalExpression "pkgs.grafana"; + description = "Grafana package to run."; + }; + + machine = lib.mkOption { + type = lib.types.str; + readOnly = true; + default = "swarm-grafana"; + description = '' + Container name. Read-only: the name appears in host paths and in + `machinectl`, so it is a fact other modules may read rather than a + knob. + ''; + }; + + domain = lib.mkOption { + type = lib.types.str; + default = "grafana.${domainBase}"; + defaultText = lib.literalExpression ''"grafana.''${services.hyperhive.swarm.domain}"''; + description = '' + Name the gateway serves this on. A sibling of the swarm's other + service names, so the swarm-services sub-CA can issue for it — see + `hive-tls.nix` for why a service name being a sibling rather than a + child decides which CA may sign it. + + ⚠️ Changing this changes the OAuth redirect URI, which authelia + matches exactly. Both sides move together because both derive from + this option; an operator who pins one by hand breaks the login. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 3000; + description = '' + Port Grafana listens on, bound to loopback only. Upstream's own + default, kept so an operator reading Grafana documentation finds + what they expect. + ''; + }; + + datasourceUrl = lib.mkOption { + type = lib.types.str; + default = "http://127.0.0.1:${toString vmCfg.port}"; + defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"''; + description = '' + Where the provisioned datasource points. Defaults to the metrics + store on this host, which is the only place it can be: that store + binds loopback, so a Grafana somewhere else could not reach it + anyway. Set explicitly if a deployment fronts VictoriaMetrics with + something that does listen wider. + ''; + }; + + oidc = { + clientId = lib.mkOption { + type = lib.types.str; + default = "swarm-grafana"; + description = '' + The authelia OIDC client id. Names the application rather than + the protocol, per the convention in + {option}`services.hyperhive.swarm.authelia.oidc.clients`. + ''; + }; + + role = lib.mkOption { + type = lib.types.enum [ + "Viewer" + "Editor" + "Admin" + ]; + default = "Admin"; + example = "Editor"; + description = '' + Grafana org role every SSO user is assigned. + + `Admin` by default, and that is a considered default rather than + a permissive one: the login form is disabled whenever SSO is + configured, so this is the *only* way anyone reaches Grafana — + a `Viewer` default would produce a swarm nobody can administer. + Passing authelia already means being an operator of this swarm; + its user store is the small, `swarmctl`-managed one. + + Lower it if a swarm ever grows read-only operators, which is a + one-line change here. + ''; + }; + }; + }; + + config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { + # The gateway name and the quick-link, both inside `cfg.enable` — that + # guard is the load-bearing part. Every hive in a swarm may know this UI + # exists, but only the host that RUNS it may claim the name; a client + # hive declaring the vhost would answer for a service it does not have. + services.hyperhive.gateway.localNames = [ cfg.domain ]; + + services.hyperhive.swarm.controller.links = [ + { + label = "Grafana"; + icon = "📊"; + url = "https://${cfg.domain}/"; + } + ]; + + # One declaration, two readers. Grafana's callback URL is format-locked + # to its own root URL; making the operator restate it in authelia's + # client list would be a second source of truth for a string whose + # mismatch is a silently rejected login. + # + # `kind` is left at its `interactive` default: a person logs in here. + services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [ + { + id = cfg.oidc.clientId; + description = "HyperHive swarm metrics"; + redirectUris = [ redirectUri ]; + } + ]; + + assertions = [ + { + # Grafana reaches the token endpoint server-to-server, so a null URL + # would become a request to `null/api/oidc/token` — a DNS failure + # several layers from its cause. Only reachable by enabling authelia + # and clearing its `url`, which is why it is an assertion and not a + # fallback. + assertion = !ssoLocal || autheliaCfg.url != null; + message = '' + services.hyperhive.swarm.grafana requires + services.hyperhive.swarm.authelia.url when authelia is enabled. + + Grafana exchanges its authorization code at + `''${url}/api/oidc/token` from inside its container. With the URL + null there is no endpoint to name. + ''; + } + ]; + + # Websockets: Grafana Live streams panel updates over one, and without + # the upgrade headers dashboards load and then never refresh — which + # reads as stale data rather than as a proxy fault. + services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { + listen = gatewayCfg.lib.listen; + extraConfig = gatewayCfg.lib.securityHeaders; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + proxyWebsockets = true; + extraConfig = '' + # Grafana builds its OAuth redirect from the ORIGINAL request. + # Without these every request looks like it arrived at 127.0.0.1 + # over plain http, and the redirect sent to authelia names a + # host the browser cannot reach. + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; + + # Order the container after the host CA service so the bind source below + # exists before nspawn sets the mount up. + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + + # The secret delivery. It runs on the HOST because that is the only place + # both container trees are addressable: they share this host's network + # namespace, which makes them feel co-located, but their filesystem roots + # are separate — Grafana cannot open a path inside authelia's tree + # however local the port looks. + # + # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container + # refuses to start when a bind source is missing, and this secret does + # not exist until authelia's first boot has minted it — so binding it + # would make Grafana wait on a file that waits on a container that starts + # after it. + systemd.services.swarm-grafana-oidc-secret = lib.mkIf ssoLocal { + description = "deliver Grafana's OIDC client secret from authelia"; + after = [ "container@${autheliaCfg.machine}.service" ]; + requires = [ "container@${autheliaCfg.machine}.service" ]; + before = [ "container@${cfg.machine}.service" ]; + wantedBy = [ "container@${cfg.machine}.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-grafana-oidc-secret"; + # ⚠️ Longer than the wait below, and that is the whole point: + # `DefaultTimeoutStartSec` is 90s, so a 120s bounded wait is killed + # by systemd at 90 — before it can emit the error naming the file it + # waited for. The timeout has to outlive the thing it is timing. + TimeoutStartSec = "180s"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + + src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.oidc.clientId}.secret"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"} + + # authelia's container is up, but its first-boot generator may still + # be minting. Bounded wait, then fail: a silent skip here produces a + # Grafana whose only login path dead-ends. + deadline=$(( SECONDS + 120 )) + while [ ! -s "$src" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "authelia has not minted $src after 120s" >&2 + exit 1 + fi + sleep 2 + done + + # Owned by Grafana's own uid, unlike the matrix sibling which lands + # root-owned: tuwunel's secret is read by `LoadCredential` as root + # before the sandbox exists, whereas Grafana expands `$__file{}` + # itself, as itself, while parsing its config. These containers set + # no `privateUsers`, so the host uid is the container uid, and both + # sides take it from the same static NixOS id. + # + # Group is root, not grafana, and that is forced rather than chosen: + # `ids.uids.grafana` is a static id but there is no `ids.gids.grafana` + # — the group's gid is allocated at activation inside the container, + # so the host cannot know it at eval time. Harmless here because 0400 + # grants the group nothing; if this mode ever widens, the gid has to + # be discovered at runtime rather than assumed. + install -D -m 0400 -o ${toString config.ids.uids.grafana} -g 0 "$src" "$dst" + ''; + }; + + containers.${cfg.machine} = { + autoStart = true; + ephemeral = false; + # Shared host netns, like every sibling swarm container: the gateway + # reaches this at 127.0.0.1:. + privateNetwork = false; + + bindMounts = { } // caTrust.bindMount; + + config = + { ... }: + { + system.stateVersion = "26.05"; + + # This container shares the host netns, so its own firewall.service + # would rewrite the HOST ruleset at every boot. The host firewall + # owns all filtering. + networking.firewall.enable = false; + # Keep the host-copied /etc/resolv.conf intact — resolvconf's + # host-tracking would regenerate it to an empty file, since the + # host's copy doesn't cross the boundary after start. + networking.resolvconf.enable = lib.mkForce false; + + # Self-signed mode: Grafana is Go, and Go's `SSL_CERT_FILE` + # *replaces* the default bundle rather than adding to it — so + # concatenate the system CAs with the bind-mounted hive CA instead + # of pointing at the CA alone, which would lose every public + # anchor. /run is tmpfs, so this is rebuilt from the current CA + # each boot rather than going stale. + # + # Without it the browser half of the login succeeds and the + # server-to-server token exchange fails with an x509 "unknown + # authority" — the same shape of failure the swarm queue hit. + systemd.services.swarm-grafana-ca-bundle = lib.mkIf useSelfSigned { + description = "assemble Grafana TLS trust bundle (system CAs + hive CA)"; + wantedBy = [ "grafana.service" ]; + before = [ "grafana.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + SyslogIdentifier = "swarm-grafana-ca-bundle"; + }; + path = [ pkgs.coreutils ]; + script = '' + set -euo pipefail + install -d -m 0755 /run/swarm-grafana-ca + cat /etc/ssl/certs/ca-certificates.crt ${caTrust.caContainerPath} \ + > ${grafanaCaBundle} + chmod 0644 ${grafanaCaBundle} + ''; + }; + + systemd.services.grafana.environment.SSL_CERT_FILE = lib.mkIf useSelfSigned grafanaCaBundle; + + services.grafana = { + enable = true; + package = cfg.package; + + settings = { + server = { + # Already upstream's default (measured), but pinned rather + # than inherited: the gateway is the only intended client, + # and this being loopback is what keeps the UI from being + # published on whatever else the host is reachable on. + http_addr = "127.0.0.1"; + http_port = cfg.port; + domain = cfg.domain; + # Grafana builds its own OAuth redirect from this. Left at + # upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it + # would name `http://:3000/`, which authelia has + # never heard of. + root_url = "https://${cfg.domain}/"; + }; + + analytics = { + reporting_enabled = false; + check_for_updates = false; + }; + + users.auto_assign_org_role = cfg.oidc.role; + + # No local password path at all when SSO is configured. This + # is not tidiness: Grafana ships an `admin`/`admin` account, + # and this vhost is on the public gateway. + auth.disable_login_form = ssoLocal; + } + // lib.optionalAttrs ssoLocal { + "auth.generic_oauth" = { + enabled = true; + name = "HyperHive"; + client_id = cfg.oidc.clientId; + # ⚠️ `$__file{}`, never the secret itself — anything else + # here is world-readable in the nix store. Note the module's + # own leak assertion does NOT cover this key (it checks + # `database.password`, `security.admin_password` and + # datasource `secureJsonData`), so nothing but this comment + # stands between a literal and the store. + client_secret = "$__file{${secretPath}}"; + scopes = "openid profile email groups"; + auth_url = "${autheliaUrl}/api/oidc/authorization"; + token_url = "${autheliaUrl}/api/oidc/token"; + api_url = "${autheliaUrl}/api/oidc/userinfo"; + use_pkce = true; + }; + }; + + provision.datasources.settings = { + apiVersion = 1; + datasources = [ + { + name = "VictoriaMetrics"; + type = "prometheus"; + uid = "swarm-victoriametrics"; + url = cfg.datasourceUrl; + access = "proxy"; + isDefault = true; + } + ]; + }; + }; + }; + }; + }; +}