diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 473535a7..cea50f2e 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -164,6 +164,12 @@ reads it, in two containers at `metrics.` and `grafana.`. Two containers rather than one so you can restart or break Grafana without taking the time-series database with it. +The Grafana container sits on the hive bridge in a network namespace of its +own and dials authelia, the secret store, the collector and both datasources +by name through it. It binds no TCP port: it listens on the unix socket +`/run/swarm-grafana/grafana.sock`, a directory only the gateway's nginx and +the container can open. + They derive together: a store with no UI is unreadable and a UI with no store is empty. To run one without the other, set it directly: diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index b9f5dc39..47f62470 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -312,8 +312,8 @@ let ) ":${toString gatewayCfg.httpsPort}"; forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}"; forwarderUnit = "swarm-journal-forwarder-oidc"; - # In the host netns: distinct from every other collector's there, which the - # assertion below checks. + # Loopback port of the forwarder's own metrics, inside the container's netns: + # no other collector shares it. The module requires a value. forwarderTelemetryPort = 8896; # Grafana's store identity, handed into the container as system credentials @@ -410,9 +410,10 @@ let # container on a switch. ensureSecretDir = "install -d -m 0755 ${lib.escapeShellArg secretDir}"; - # Shared host netns, like every sibling swarm container: the gateway - # reaches this at 127.0.0.1:. - privateNetwork = false; + # A netns of its own on the hive bridge: the container dials the store, + # authelia, the swarm collector and both datasources through the bridge, and + # the gateway reaches Grafana over its socket, so no TCP port is bound. + privateNetwork = true; storeRetry = import ./lib/store-retry.nix { }; storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; in @@ -554,6 +555,9 @@ in services.hyperhive.gateway.enable = lib.mkDefault true; services.hyperhive.gateway.dns.enable = lib.mkDefault true; + # The hive bridge carries the container's DHCP lease and its resolver. + services.hyperhive.network.enable = lib.mkDefault true; + # Registering the client is NOT here: it has to happen on the host that # runs authelia, and this whole block is gated on the host that runs # Grafana. ./glue-grafana-oidc-client.nix does it. @@ -611,21 +615,6 @@ in it in swarm-otel-service.nix. ''; } - { - assertion = - !(lib.elem forwarderTelemetryPort [ - hyperhiveCfg.otel.telemetryPort - hyperhiveCfg.swarm.otel.telemetryPort - hyperhiveCfg.swarm.bao.otel.telemetryPort - hyperhiveCfg.swarm.nats.otel.telemetryPort - ]); - message = '' - Grafana's journal forwarder's telemetry port (${toString forwarderTelemetryPort}) - collides with another collector's in the host netns: the hive - collector's, the swarm collector's, the store forwarder's or the - queue forwarder's. - ''; - } ]; # Websockets: Grafana Live streams panel updates over one, and without @@ -1181,6 +1170,7 @@ in # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix). extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles; inherit privateNetwork; + hostBridge = networkCfg.bridgeName; # The socket directory, shared with the host so nginx can reach in. # @@ -1238,6 +1228,11 @@ in services.hyperhive.swarmContainer = { inherit privateNetwork; }; + # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns` + # owns resolv.conf, so dhcpcd must not write it over that unit. + networking.interfaces.eth0.useDHCP = true; + networking.dhcpcd.extraConfig = "nohook resolv.conf"; + # Authenticated as the forwarder's own OIDC client, with Grafana's # store identity: one identity for the whole container. It is a # system credential, so the fetch unit loads it. diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix index 19c1fd58..105bcdd0 100644 --- a/nix/module-eval/swarm-services-switch.nix +++ b/nix/module-eval/swarm-services-switch.nix @@ -171,8 +171,8 @@ let # Every service container at once: the services-only host plus the CI # runner. The CI runner, the review classifier, the builder, the claude login - # minter, the metrics store, the homeserver and authelia are the seven containers on a - # netns of their own. + # minter, the metrics store, the homeserver, authelia and Grafana are the eight + # containers on a netns of their own. serviceContainersWithCi = hive { deploy.hive-controller.enable = false; deploy.allSwarmServices = true; @@ -621,7 +621,7 @@ let let shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers); in - lib.length shared == 6 + lib.length shared == 5 && lib.all ( c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable ) shared; @@ -761,6 +761,37 @@ let && bridge.serviceConfig.Group == "gateway-nginx" && lib.elem dir unit.serviceConfig.ReadWritePaths; } + { + # On the bridge by DHCP, listening on a socket (protocol `socket` ignores the + # upstream-default `http_port`, which stays in the settings): the + # gateway's locations (UI, metrics, websocket upgrade) all forward to it. + # `forwarded != []` keeps an empty filter from passing. + name = "grafana runs on a netns of its own and the gateway reaches it over its socket"; + ok = + let + host = serviceContainersWithCi; + net = host.services.hyperhive.network; + c = host.containers.swarm-grafana; + dir = "/run/swarm-grafana"; + sock = "${dir}/grafana.sock"; + locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.grafana.domain}.locations; + forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) ( + map (l: l.proxyPass) (lib.attrValues locs) + ); + server = c.config.services.grafana.settings.server; + in + c.privateNetwork + && c.hostBridge == net.bridgeName + && c.hostAddress == null + && c.localAddress == null + && c.config.networking.interfaces.eth0.useDHCP + && c.config.networking.firewall.enable + && server.protocol == "socket" + && server.socket == sock + && forwarded != [ ] + && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded + && c.bindMounts.${dir}.hostPath == dir; + } { # The DynamicUser layout keeps the history under /var/lib/private; the # static user's StateDirectory is the plain path. The move runs as its