swarm-grafana: run in its own network namespace

docs/swarm/services.md

@@ -164,6 +164,12 @@ reads it, in two containers at `metrics.<swarm-domain>` and
 `grafana.<swarm-domain>`. Two containers rather than one so you can
 restart or break Grafana without taking the time-series database with it.
 
+The Grafana container sits on the hive bridge in a network namespace of its
+own and dials authelia, the secret store, the collector and both datasources
+by name through it. It binds no TCP port: it listens on the unix socket
+`/run/swarm-grafana/grafana.sock`, a directory only the gateway's nginx and
+the container can open.
+
 They derive together: a store with no UI is unreadable and a UI with no
 store is empty. To run one without the other, set it directly:
 

nix/host-modules/swarm-grafana.nix

@@ -312,8 +312,8 @@ let
   ) ":${toString gatewayCfg.httpsPort}";
   forwarderEndpoint = "https://${hyperhiveCfg.swarm.otel.domain}${gatewayPortSuffix}/${cfg.machine}";
   forwarderUnit = "swarm-journal-forwarder-oidc";
-  # In the host netns: distinct from every other collector's there, which the
+  # Loopback port of the forwarder's own metrics, inside the container's netns:
-  # assertion below checks.
+  # no other collector shares it. The module requires a value.
   forwarderTelemetryPort = 8896;
 
   # Grafana's store identity, handed into the container as system credentials
@@ -410,9 +410,10 @@ let
   # container on a switch.
   ensureSecretDir = "install -d -m 0755 ${lib.escapeShellArg secretDir}";
 
-  # Shared host netns, like every sibling swarm container: the gateway
+  # A netns of its own on the hive bridge: the container dials the store,
-  # reaches this at 127.0.0.1:<port>.
+  # authelia, the swarm collector and both datasources through the bridge, and
-  privateNetwork = false;
+  # the gateway reaches Grafana over its socket, so no TCP port is bound.
+  privateNetwork = true;
   storeRetry = import ./lib/store-retry.nix { };
   storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
 in
@@ -554,6 +555,9 @@ in
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
+    # The hive bridge carries the container's DHCP lease and its resolver.
+    services.hyperhive.network.enable = lib.mkDefault true;
+
     # Registering the client is NOT here: it has to happen on the host that
     # runs authelia, and this whole block is gated on the host that runs
     # Grafana. ./glue-grafana-oidc-client.nix does it.
@@ -611,21 +615,6 @@ in
           it in swarm-otel-service.nix.
         '';
       }
-      {
-        assertion =
-          !(lib.elem forwarderTelemetryPort [
-            hyperhiveCfg.otel.telemetryPort
-            hyperhiveCfg.swarm.otel.telemetryPort
-            hyperhiveCfg.swarm.bao.otel.telemetryPort
-            hyperhiveCfg.swarm.nats.otel.telemetryPort
-          ]);
-        message = ''
-          Grafana's journal forwarder's telemetry port (${toString forwarderTelemetryPort})
-          collides with another collector's in the host netns: the hive
-          collector's, the swarm collector's, the store forwarder's or the
-          queue forwarder's.
-        '';
-      }
     ];
 
     # Websockets: Grafana Live streams panel updates over one, and without
@@ -1181,6 +1170,7 @@ in
       # container's own forwarder reads it (../container-modules/swarm-journal-forwarder.nix).
       extraFlags = lib.mapAttrsToList (id: source: "--load-credential=${id}:${source}") storeFiles;
       inherit privateNetwork;
+      hostBridge = networkCfg.bridgeName;
 
       # The socket directory, shared with the host so nginx can reach in.
       #
@@ -1238,6 +1228,11 @@ in
 
           services.hyperhive.swarmContainer = { inherit privateNetwork; };
 
+          # The bridge address comes from the gateway's DHCP. `swarm-bridge-dns`
+          # owns resolv.conf, so dhcpcd must not write it over that unit.
+          networking.interfaces.eth0.useDHCP = true;
+          networking.dhcpcd.extraConfig = "nohook resolv.conf";
+
           # Authenticated as the forwarder's own OIDC client, with Grafana's
           # store identity: one identity for the whole container. It is a
           # system credential, so the fetch unit loads it.

nix/module-eval/swarm-services-switch.nix

@@ -171,8 +171,8 @@ let
 
   # Every service container at once: the services-only host plus the CI
   # runner. The CI runner, the review classifier, the builder, the claude login
-  # minter, the metrics store, the homeserver and authelia are the seven containers on a
+  # minter, the metrics store, the homeserver, authelia and Grafana are the eight
-  # netns of their own.
+  # containers on a netns of their own.
   serviceContainersWithCi = hive {
     deploy.hive-controller.enable = false;
     deploy.allSwarmServices = true;
@@ -621,7 +621,7 @@ let
         let
           shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
         in
-        lib.length shared == 6
+        lib.length shared == 5
         && lib.all (
           c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
         ) shared;
@@ -761,6 +761,37 @@ let
         && bridge.serviceConfig.Group == "gateway-nginx"
         && lib.elem dir unit.serviceConfig.ReadWritePaths;
     }
+    {
+      # On the bridge by DHCP, listening on a socket (protocol `socket` ignores the
+      # upstream-default `http_port`, which stays in the settings): the
+      # gateway's locations (UI, metrics, websocket upgrade) all forward to it.
+      # `forwarded != []` keeps an empty filter from passing.
+      name = "grafana runs on a netns of its own and the gateway reaches it over its socket";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          net = host.services.hyperhive.network;
+          c = host.containers.swarm-grafana;
+          dir = "/run/swarm-grafana";
+          sock = "${dir}/grafana.sock";
+          locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.grafana.domain}.locations;
+          forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
+            map (l: l.proxyPass) (lib.attrValues locs)
+          );
+          server = c.config.services.grafana.settings.server;
+        in
+        c.privateNetwork
+        && c.hostBridge == net.bridgeName
+        && c.hostAddress == null
+        && c.localAddress == null
+        && c.config.networking.interfaces.eth0.useDHCP
+        && c.config.networking.firewall.enable
+        && server.protocol == "socket"
+        && server.socket == sock
+        && forwarded != [ ]
+        && lib.all (lib.hasPrefix "http://unix:${sock}:") forwarded
+        && c.bindMounts.${dir}.hostPath == dir;
+    }
     {
       # The DynamicUser layout keeps the history under /var/lib/private; the
       # static user's StateDirectory is the plain path. The move runs as its