# The swarm's metrics UI: one Grafana for the whole swarm, in a # `swarm-grafana` nixos-container next to the VictoriaMetrics store it reads. # # Two containers rather than one, on the operator's call: Grafana can be # restarted, reconfigured or broken without taking the TSDB down with it. # They are a pair, not a unit. # # Dashboards are provisioned from files in this repo, deliberately not created # in the UI: the store path is immutable, so Grafana is told not to accept UI # edits it would lose on the next rebuild. { pkgs, lib, config, ... }: let cfg = config.services.hyperhive.swarm.grafana; networkCfg = config.services.hyperhive.network; hyperhiveCfg = config.services.hyperhive; gatewayCfg = hyperhiveCfg.gateway; tlsCfg = hyperhiveCfg.tls; autheliaCfg = hyperhiveCfg.swarm.authelia; vmCfg = hyperhiveCfg.swarm.victoriametrics; swarmDomain = hyperhiveCfg.swarm.domain; caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # Spelled ONCE. The provisioned datasource declares it and every panel in the # provisioned dashboards references it; a second literal would be free to # drift, and the failure is a dashboard that renders with empty panels rather # than an error. datasourceUid = "swarm-victoriametrics"; # The shipped dashboards carry `@datasourceUid@` where a real deployment needs # the uid above. They are substituted here rather than committed with the # literal so the single binding stays single. renderDashboard = name: pkgs.writeText name ( builtins.replaceStrings [ "@datasourceUid@" ] [ datasourceUid ] ( builtins.readFile (./swarm-grafana/dashboards + "/${name}") ) ); # Grafana's file provider wants a DIRECTORY to scan, so the rendered files are # collected into one. dashboardDir = pkgs.linkFarm "hyperhive-grafana-dashboards" ( map (name: { inherit name; path = renderDashboard name; }) [ "agents.json" "claude-usage.json" ] ); # Total on a null swarm domain for the same reason every sibling module is: # the required-domain assertion in hive-network.nix should be what an # operator sees, not a coercion error from here. domainBase = if swarmDomain == null then "invalid" else swarmDomain; # The all-local case: this host runs BOTH Grafana and the swarm's authelia, # so the minted secret can be moved without an operator. Same split the # forge and matrix modules document. ssoLocal = cfg.enable && autheliaCfg.enable; autheliaUrl = toString autheliaCfg.url; # Where the plaintext lands inside the container. Under /var/lib rather # than /run: Grafana may start before the delivery unit on a later boot, # and a secret that evaporates on reboot turns a working login into an # intermittent one. secretPath = "/var/lib/grafana-oidc/${cfg.oidc.clientId}.secret"; # Grafana's own datasource-encryption key. Generated in-container (see the # unit below) because nothing outside the container ever reads it — unlike # the OIDC secret above, whose other reader is authelia's container. secretKeyPath = "/var/lib/grafana-secret/secret_key"; socketPath = "${cfg.socketDir}/grafana.sock"; # Both static NixOS ids, and both checked rather than assumed: nginx has # `ids.gids.nginx = 60`, Grafana has `ids.uids.grafana = 196` — but there is # deliberately no `ids.gids.grafana`, which is why the OIDC-secret unit # below lands its file root-group at 0400 instead of reaching for one. nginxGid = config.ids.gids.nginx; grafanaUid = config.ids.uids.grafana; # Format-locked by Grafana: the generic OAuth callback is always # `/login/generic_oauth`. Declared once here and read by both # the authelia client and Grafana itself. redirectUri = "https://${cfg.domain}/login/generic_oauth"; in { options.services.hyperhive.swarm.grafana = { enable = lib.mkOption { type = lib.types.bool; default = false; description = '' Run the swarm's metrics UI on this host. Off by default and not derived from {option}`services.hyperhive.enable`: a swarm has one Grafana, so enabling it is a decision about swarm topology rather than about whether hyperhive is installed. ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.grafana; defaultText = lib.literalExpression "pkgs.grafana"; description = "Grafana package to run."; }; machine = lib.mkOption { type = lib.types.str; readOnly = true; default = "swarm-grafana"; description = '' Container name. Read-only: the name appears in host paths and in `machinectl`, so it is a fact other modules may read rather than a knob. ''; }; domain = lib.mkOption { type = lib.types.str; default = "grafana.${domainBase}"; defaultText = lib.literalExpression ''"grafana.''${services.hyperhive.swarm.domain}"''; description = '' Name the gateway serves this on. A sibling of the swarm's other service names, so the swarm-services sub-CA can issue for it — see `hive-tls.nix` for why a service name being a sibling rather than a child decides which CA may sign it. ⚠️ Changing this changes the OAuth redirect URI, which authelia matches exactly. Both sides move together because both derive from this option; an operator who pins one by hand breaks the login. ''; }; socketDir = lib.mkOption { type = lib.types.str; default = "/run/swarm-grafana"; description = '' Directory holding the unix socket Grafana serves on, shared between the host (where nginx runs) and the container (where Grafana runs). ⚠️ **Grafana takes no TCP port at all, and that is the point.** Every swarm service container shares the host's network namespace, so a port is a swarm-wide resource that two modules can silently both claim — which is exactly what happened: Grafana defaulted to upstream's 3000, so does the forge, and `grafana.` served the forge with no bind error and nothing in any log. A socket has a path, and a path collision is a build-time conflict rather than a runtime coin toss. ''; }; datasourceUrl = lib.mkOption { type = lib.types.str; default = "http://127.0.0.1:${toString vmCfg.port}"; defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"''; description = '' Where the provisioned datasource points. Defaults to the metrics store on this host, which is the only place it can be: that store binds loopback, so a Grafana somewhere else could not reach it anyway. Set explicitly if a deployment fronts VictoriaMetrics with something that does listen wider. ''; }; oidc = { clientId = lib.mkOption { type = lib.types.str; default = "swarm-grafana"; description = '' The authelia OIDC client id. Names the application rather than the protocol, per the convention in {option}`services.hyperhive.swarm.authelia.oidc.clients`. ''; }; role = lib.mkOption { type = lib.types.enum [ "Viewer" "Editor" "Admin" ]; default = "Admin"; example = "Editor"; description = '' Grafana org role every SSO user is assigned. `Admin` by default, and that is a considered default rather than a permissive one: the login form is disabled whenever SSO is configured, so this is the *only* way anyone reaches Grafana — a `Viewer` default would produce a swarm nobody can administer. Passing authelia already means being an operator of this swarm; its user store is the small, `swarmctl`-managed one. Lower it if a swarm ever grows read-only operators, which is a one-line change here. ''; }; }; plugins = lib.mkOption { type = lib.types.listOf lib.types.package; default = [ pkgs.grafanaPlugins.grafana-metricsdrilldown-app ]; defaultText = lib.literalExpression "[ pkgs.grafanaPlugins.grafana-metricsdrilldown-app ]"; example = lib.literalExpression "[ pkgs.grafanaPlugins.grafana-piechart-panel ]"; description = '' Grafana plugins to install, as packages. Declarative rather than installed through the UI, which is the only shape that works here: plugin management is **server-admin** scoped, and on an SSO hive nobody holds that role — `users.auto_assign_org_role` grants an *org* role, and the built-in local admin that does hold server admin cannot log in because the login form is disabled whenever SSO is configured. That is a deliberate pair of decisions rather than an oversight, and this option is the way through it: plugins live in the store and in git, so they survive a container rebuild and a state reset, and the container needs no runtime egress to grafana.com. ⚠️ This list is the ONLY way a plugin reaches this Grafana. Setting it at all disables upstream's `preinstall` mechanism (see the `declarativePlugins` comment below), which is what normally fetches the Drilldown apps on first start — so a plugin left off this list is absent, not merely un-pinned. Metrics Drilldown is on by default because this swarm's Grafana exists to read VictoriaMetrics, and the queryless metrics browser is the part of that people actually reach for. Its Logs / Traces / Profiles siblings are deliberately NOT here: they front Loki, Tempo and Pyroscope, none of which this swarm runs, so they would install a UI for a backend that does not answer. ''; }; }; config = lib.mkIf (hyperhiveCfg.enable && cfg.enable) { # The gateway name and the quick-link, both inside `cfg.enable` — that # guard is the load-bearing part. Every hive in a swarm may know this UI # exists, but only the host that RUNS it may claim the name; a client # hive declaring the vhost would answer for a service it does not have. services.hyperhive.gateway.localNames = [ cfg.domain ]; services.hyperhive.swarm.controller.links = [ { label = "Grafana"; icon = "📊"; url = "https://${cfg.domain}/"; } ]; # One declaration, two readers. Grafana's callback URL is format-locked # to its own root URL; making the operator restate it in authelia's # client list would be a second source of truth for a string whose # mismatch is a silently rejected login. # # `kind` is left at its `interactive` default: a person logs in here. services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [ { id = cfg.oidc.clientId; description = "HyperHive swarm metrics"; redirectUris = [ redirectUri ]; } ]; assertions = [ { # Grafana reaches the token endpoint server-to-server, so a null URL # would become a request to `null/api/oidc/token` — a DNS failure # several layers from its cause. Only reachable by enabling authelia # and clearing its `url`, which is why it is an assertion and not a # fallback. assertion = !ssoLocal || autheliaCfg.url != null; message = '' services.hyperhive.swarm.grafana requires services.hyperhive.swarm.authelia.url when authelia is enabled. Grafana exchanges its authorization code at `''${url}/api/oidc/token` from inside its container. With the URL null there is no endpoint to name. ''; } ]; # Websockets: Grafana Live streams panel updates over one, and without # the upgrade headers dashboards load and then never refresh — which # reads as stale data rather than as a proxy fault. services.nginx.virtualHosts."${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; locations."/" = { proxyPass = "http://unix:${socketPath}:/"; proxyWebsockets = true; extraConfig = '' # Grafana builds its OAuth redirect from the ORIGINAL request. # Without these every request looks like it arrived at 127.0.0.1 # over plain http, and the redirect sent to authelia names a # host the browser cannot reach. proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; ''; }; }; # Order the container after the host CA service so the bind source below # exists before nspawn sets the mount up. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; # The socket directory, created host-side before the container starts. # nixos-container refuses to start when a bind source is missing, so this # rule is a prerequisite of the mount rather than a tidiness measure. # # Owned by Grafana's uid so it can create the socket, group nginx so the # gateway can traverse; `0750` keeps everything else out. Numeric ids # because the host has no `grafana` account — Grafana lives in the # container, and only the number crosses that boundary. systemd.tmpfiles.rules = [ "d ${cfg.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" ]; # The secret delivery. It runs on the HOST because that is the only place # both container trees are addressable: they share this host's network # namespace, which makes them feel co-located, but their filesystem roots # are separate — Grafana cannot open a path inside authelia's tree # however local the port looks. # # ⚠️ Deliberately a copy and not a `bindMounts` entry. nixos-container # refuses to start when a bind source is missing, and this secret does # not exist until authelia's first boot has minted it — so binding it # would make Grafana wait on a file that waits on a container that starts # after it. systemd.services.swarm-grafana-oidc-secret = lib.mkIf ssoLocal { description = "deliver Grafana's OIDC client secret from authelia"; after = [ "container@${autheliaCfg.machine}.service" ]; requires = [ "container@${autheliaCfg.machine}.service" ]; before = [ "container@${cfg.machine}.service" ]; wantedBy = [ "container@${cfg.machine}.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; SyslogIdentifier = "swarm-grafana-oidc-secret"; # ⚠️ Longer than the wait below, and that is the whole point: # `DefaultTimeoutStartSec` is 90s, so a 120s bounded wait is killed # by systemd at 90 — before it can emit the error naming the file it # waited for. The timeout has to outlive the thing it is timing. TimeoutStartSec = "180s"; }; path = [ pkgs.coreutils ]; script = '' set -euo pipefail src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.oidc.clientId}.secret"} dst=${lib.escapeShellArg "/var/lib/nixos-containers/${cfg.machine}${secretPath}"} # authelia's container is up, but its first-boot generator may still # be minting. Bounded wait, then fail: a silent skip here produces a # Grafana whose only login path dead-ends. deadline=$(( SECONDS + 120 )) while [ ! -s "$src" ]; do if [ "$SECONDS" -ge "$deadline" ]; then echo "authelia has not minted $src after 120s" >&2 exit 1 fi sleep 2 done # Owned by Grafana's own uid, unlike the matrix sibling which lands # root-owned: tuwunel's secret is read by `LoadCredential` as root # before the sandbox exists, whereas Grafana expands `$__file{}` # itself, as itself, while parsing its config. These containers set # no `privateUsers`, so the host uid is the container uid, and both # sides take it from the same static NixOS id. # # Group is root, not grafana, and that is forced rather than chosen: # `ids.uids.grafana` is a static id but there is no `ids.gids.grafana` # — the group's gid is allocated at activation inside the container, # so the host cannot know it at eval time. Harmless here because 0400 # grants the group nothing; if this mode ever widens, the gid has to # be discovered at runtime rather than assumed. install -D -m 0400 -o ${toString config.ids.uids.grafana} -g 0 "$src" "$dst" ''; }; containers.${cfg.machine} = { autoStart = true; ephemeral = false; # Shared host netns, like every sibling swarm container: the gateway # reaches this at 127.0.0.1:. privateNetwork = false; # The socket directory, shared with the host so nginx can reach in. # # ⚠️ It has to be a bind mount rather than a path both sides happen to # know: a container's `/run` is an nspawn tmpfs, so it is NOT visible # from the host at `/var/lib/nixos-containers//run`. The host # creates the directory (tmpfiles rule below, which also fixes # ownership) and nspawn mounts it in — the same shape the per-agent # `/run/hive-agent//web.sock` already uses. bindMounts = { ${cfg.socketDir} = { hostPath = cfg.socketDir; isReadOnly = false; }; } // caTrust.bindMount; config = { ... }: { imports = [ (import ./swarm-container-resolver.nix { inherit (networkCfg) bridgeIp; dnsConsumers = [ "grafana.service" ]; }) # Grafana is Go, and `SSL_CERT_FILE` *replaces* the default store # rather than adding to it — so it needs the system CAs and the # hive CA concatenated, not the CA alone. Without it the browser # half of the login succeeds and the server-to-server token # exchange fails `x509: unknown authority`. (caTrust.trustBundle { inherit pkgs; name = cfg.machine; consumers = [ "grafana" ]; }) ]; system.stateVersion = "26.05"; # This container shares the host netns, so its own firewall.service # would rewrite the HOST ruleset at every boot. The host firewall # owns all filtering. networking.firewall.enable = false; # resolvconf stays off because the resolver unit imported above # owns /etc/resolv.conf. Leaving it on would let host-tracking # regenerate the file empty, since the host's copy doesn't cross # the boundary after start. networking.resolvconf.enable = lib.mkForce false; # Grafana's `secret_key` has **no default in nixpkgs** and an # assertion refuses the build without one — which is how the first # version of this module broke a deploy. It signs the datasource # secrets in Grafana's own database. # # Generated in-container on first boot, like authelia's own keys and # for the same reason: nothing outside this container ever reads it, # which makes in-container generation right rather than merely # easier. (The OIDC client secret is delivered host-side precisely # because it has a second reader.) # # ⚠️ Generated ONCE and kept — the `-s` guard is load-bearing. # Rotating this key does not re-encrypt what it already encrypted, so # a fresh key on every boot would leave Grafana unable to decrypt its # own stored datasource secrets. Under /var/lib, never /run. systemd.services.swarm-grafana-secret-key = { description = "generate Grafana's datasource encryption key on first boot"; wantedBy = [ "grafana.service" ]; before = [ "grafana.service" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; # Same User/Group/StateDirectory as grafana itself, so systemd # creates the directory owned by the account that has to read # the file — no chown, no mode juggling. User = "grafana"; Group = "grafana"; StateDirectory = "grafana-secret"; StateDirectoryMode = "0700"; SyslogIdentifier = "swarm-grafana-secret-key"; }; path = [ pkgs.openssl pkgs.coreutils ]; script = '' set -euo pipefail key=${lib.escapeShellArg secretKeyPath} if [ ! -s "$key" ]; then umask 077 openssl rand -hex 32 > "$key" echo "generated Grafana's datasource encryption key" fi ''; }; # The group Grafana chgrps its socket to. Declared here because a # container has its own user database — the host knowing gid 60 as # `nginx` means nothing in here, only the number crosses. # # The name is local and arbitrary; the gid is the contract. users.groups.gateway-nginx.gid = nginxGid; users.users.grafana.extraGroups = [ "gateway-nginx" ]; services.grafana = { enable = true; package = cfg.package; # Passed through unconditionally, empty default included. # Upstream distinguishes `null` from `[ ]`, and both differences # favour always handing it a list: # # - `null` points the plugin path at grafana's mutable # `/plugins`; any list points it at a store path. # Switching on the day someone adds their first plugin would # bury that change inside an unrelated one. # - upstream defaults its plugin update-check to # `declarativePlugins == null`, so a list also stops the # container phoning grafana.com. That is the no-runtime-egress # property this option exists for — it should not arrive only # once a plugin happens to be listed. # # - 🩸 and the one this comment used to miss: it also sets # upstream's `plugins.preinstall_disabled` (nixpkgs' # grafana.nix defaults it to `declarativePlugins != null`, # saying preinstall "causes issues with declarativePlugins"). # Preinstall is what fetches the Drilldown apps on first # start, so claiming this directory silently removes them. # That is why the option's default is not empty. # # The claim that used to sit here — "nothing is taken over on a # hive with no plugins, manual installation is impossible anyway" # — reasoned only about plugins a PERSON installs. Upstream # installs some itself, and those are the ones that went missing. declarativePlugins = cfg.plugins; settings = { server = { # Already upstream's default (measured), but pinned rather # than inherited: the gateway is the only intended client, # and this being loopback is what keeps the UI from being # published on whatever else the host is reachable on. # A unix socket, not a port. `socket_gid` is nginx's, so the # gateway can connect; `0660` keeps everyone else out. # # ⚠️ Grafana chowns the socket to `socket_gid`, and a # non-root process may only chgrp to a group it is a MEMBER # of — hence the group declared for `grafana` in this # container's own user database below. Without that # membership Grafana starts, fails the chown, and the socket # is unreachable by nginx with nothing obviously wrong. protocol = "socket"; socket = socketPath; socket_gid = nginxGid; socket_mode = "0660"; domain = cfg.domain; # Grafana builds its own OAuth redirect from this. Left at # upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it # would name `http://:3000/`, which authelia has # never heard of. root_url = "https://${cfg.domain}/"; }; analytics = { reporting_enabled = false; check_for_updates = false; }; users.auto_assign_org_role = cfg.oidc.role; # ⚠️ `$__file{}`, and required rather than optional: nixpkgs # dropped the default and asserts on null, so without this the # whole host fails to build. The unit below generates it. security.secret_key = "$__file{${secretKeyPath}}"; # No local password path at all when SSO is configured. This # is not tidiness: Grafana ships an `admin`/`admin` account, # and this vhost is on the public gateway. auth.disable_login_form = ssoLocal; } // lib.optionalAttrs ssoLocal { "auth.generic_oauth" = { enabled = true; name = "HyperHive"; client_id = cfg.oidc.clientId; # ⚠️ `$__file{}`, never the secret itself — anything else # here is world-readable in the nix store. Note the module's # own leak assertion does NOT cover this key (it checks # `database.password`, `security.admin_password` and # datasource `secureJsonData`), so nothing but this comment # stands between a literal and the store. client_secret = "$__file{${secretPath}}"; scopes = "openid profile email groups"; auth_url = "${autheliaUrl}/api/oidc/authorization"; token_url = "${autheliaUrl}/api/oidc/token"; api_url = "${autheliaUrl}/api/oidc/userinfo"; use_pkce = true; }; }; provision.datasources.settings = { apiVersion = 1; datasources = [ { name = "VictoriaMetrics"; type = "prometheus"; uid = datasourceUid; url = cfg.datasourceUrl; access = "proxy"; isDefault = true; } ]; }; provision.dashboards.settings = { apiVersion = 1; providers = [ { name = "hyperhive"; type = "file"; # Read-only in the UI: the store path is immutable, so an edit # saved here would be silently discarded on the next rebuild. # Better to refuse the edit than to lose it. allowUiUpdates = false; options.path = dashboardDir; } ]; }; }; }; }; }; }