diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index d5eaf15b..8332a6ee 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -46,6 +46,15 @@ let # the OIDC secret above, whose other reader is authelia's container. secretKeyPath = "/var/lib/grafana-secret/secret_key"; + socketPath = "${cfg.socketDir}/grafana.sock"; + + # Both static NixOS ids, and both checked rather than assumed: nginx has + # `ids.gids.nginx = 60`, Grafana has `ids.uids.grafana = 196` — but there is + # deliberately no `ids.gids.grafana`, which is why the OIDC-secret unit + # below lands its file root-group at 0400 instead of reaching for one. + nginxGid = config.ids.gids.nginx; + grafanaUid = config.ids.uids.grafana; + # Format-locked by Grafana: the generic OAuth callback is always # `/login/generic_oauth`. Declared once here and read by both # the authelia client and Grafana itself. @@ -100,13 +109,22 @@ in ''; }; - port = lib.mkOption { - type = lib.types.port; - default = 3000; + socketDir = lib.mkOption { + type = lib.types.str; + default = "/run/swarm-grafana"; description = '' - Port Grafana listens on, bound to loopback only. Upstream's own - default, kept so an operator reading Grafana documentation finds - what they expect. + Directory holding the unix socket Grafana serves on, shared between + the host (where nginx runs) and the container (where Grafana runs). + + ⚠️ **Grafana takes no TCP port at all, and that is the point.** Every + swarm service container shares the host's network namespace, so a + port is a swarm-wide resource that two modules can silently both + claim — which is exactly what happened: Grafana defaulted to + upstream's 3000, so does the forge, and `grafana.` + served the forge with no bind error and nothing in any log. + + A socket has a path, and a path collision is a build-time conflict + rather than a runtime coin toss. ''; }; @@ -214,7 +232,7 @@ in listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; locations."/" = { - proxyPass = "http://127.0.0.1:${toString cfg.port}/"; + proxyPass = "http://unix:${socketPath}:/"; proxyWebsockets = true; extraConfig = '' # Grafana builds its OAuth redirect from the ORIGINAL request. @@ -232,6 +250,18 @@ in # exists before nspawn sets the mount up. systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + # The socket directory, created host-side before the container starts. + # nixos-container refuses to start when a bind source is missing, so this + # rule is a prerequisite of the mount rather than a tidiness measure. + # + # Owned by Grafana's uid so it can create the socket, group nginx so the + # gateway can traverse; `0750` keeps everything else out. Numeric ids + # because the host has no `grafana` account — Grafana lives in the + # container, and only the number crosses that boundary. + systemd.tmpfiles.rules = [ + "d ${cfg.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" + ]; + # The secret delivery. It runs on the HOST because that is the only place # both container trees are addressable: they share this host's network # namespace, which makes them feel co-located, but their filesystem roots @@ -302,7 +332,21 @@ in # reaches this at 127.0.0.1:. privateNetwork = false; - bindMounts = { } // caTrust.bindMount; + # The socket directory, shared with the host so nginx can reach in. + # + # ⚠️ It has to be a bind mount rather than a path both sides happen to + # know: a container's `/run` is an nspawn tmpfs, so it is NOT visible + # from the host at `/var/lib/nixos-containers//run`. The host + # creates the directory (tmpfiles rule below, which also fixes + # ownership) and nspawn mounts it in — the same shape the per-agent + # `/run/hive-agent//web.sock` already uses. + bindMounts = { + ${cfg.socketDir} = { + hostPath = cfg.socketDir; + isReadOnly = false; + }; + } + // caTrust.bindMount; config = { ... }: @@ -395,6 +439,14 @@ in ''; }; + # The group Grafana chgrps its socket to. Declared here because a + # container has its own user database — the host knowing gid 60 as + # `nginx` means nothing in here, only the number crosses. + # + # The name is local and arbitrary; the gid is the contract. + users.groups.gateway-nginx.gid = nginxGid; + users.users.grafana.extraGroups = [ "gateway-nginx" ]; + services.grafana = { enable = true; package = cfg.package; @@ -405,8 +457,19 @@ in # than inherited: the gateway is the only intended client, # and this being loopback is what keeps the UI from being # published on whatever else the host is reachable on. - http_addr = "127.0.0.1"; - http_port = cfg.port; + # A unix socket, not a port. `socket_gid` is nginx's, so the + # gateway can connect; `0660` keeps everyone else out. + # + # ⚠️ Grafana chowns the socket to `socket_gid`, and a + # non-root process may only chgrp to a group it is a MEMBER + # of — hence the group declared for `grafana` in this + # container's own user database below. Without that + # membership Grafana starts, fails the chown, and the socket + # is unreachable by nginx with nothing obviously wrong. + protocol = "socket"; + socket = socketPath; + socket_gid = nginxGid; + socket_mode = "0660"; domain = cfg.domain; # Grafana builds its own OAuth redirect from this. Left at # upstream's `%(protocol)s://%(domain)s:%(http_port)s/` it