swarm-logs-auth: run as a static user so nginx reaches its socket

With DynamicUser=true and RuntimeDirectoryPreserve=yes, systemd creates
the unit's runtime directory under /run/private/ (0700 root:root) and
symlinks /run/swarm-logs-auth to it (systemd.exec(5),
RuntimeDirectoryPreserve=). nginx runs as the unprivileged nginx user, so
its connect() to /run/swarm-logs-auth/auth.sock failed with EACCES, the
auth subrequest ended in a 502, and auth_request answered every agent
bearer on /select/logsql/ with a 500.

The unit now runs as a static swarm-logs-auth system user and group, the
shape swarm-controller already uses for its 0751 + Preserve socket dir.
RestrictSUIDSGID and RemoveIPC, which DynamicUser implied and the unit
did not set itself, are set explicitly.

module-eval-core-toggle gains a case that fails when any unit creating
the runtime directory of a unix: upstream of nginx combines DynamicUser
with RuntimeDirectoryPreserve. docs/process/gotchas.md records the trap.

Fixes #5235

docs/process/gotchas.md

@@ -75,6 +75,17 @@ the path entry resolves to `/run/wrappers/bin/bin` instead.
 hive-c0re restarts. Without it, every restart wipes bind sources and
 existing containers fail to start.
 
+### `DynamicUser` + `RuntimeDirectoryPreserve` hides the socket from nginx
+
+With both set, systemd creates the runtime directory under `/run/private/`
+(`0700 root:root`) and symlinks `/run/<name>` to it (`systemd.exec(5)`,
+`RuntimeDirectoryPreserve=`). No unprivileged user can traverse that, so
+nginx's `connect()` to a socket in it fails with `EACCES` whatever the
+directory and socket modes are. A unit serving a socket to another user
+runs as a static user instead (`swarm-controller`, `swarm-logs-auth`).
+`module-eval-core-toggle` fails on any `unix:` upstream of nginx owned by
+such a unit.
+
 ### `RestrictAddressFamilies` fails as "Address family not supported by protocol"
 
 A unit whose `RestrictAddressFamilies` omits a family gets `EAFNOSUPPORT`

nix/host-modules/swarm-victorialogs.nix

@@ -372,6 +372,13 @@ in
     # The agents' check on `/select/logsql/`. On the host rather than in the
     # store's container: it needs nothing of the store, only nginx's reach to
     # its socket and its own store identity.
+    users.users.${logsAuthUnit} = {
+      isSystemUser = true;
+      group = logsAuthUnit;
+      description = "swarm log store auth_request responder";
+    };
+    users.groups.${logsAuthUnit} = { };
+
     systemd.services.${logsAuthUnit} = {
       description = "swarm log store auth_request responder for agent credentials";
       wantedBy = [ "multi-user.target" ];
@@ -391,7 +398,11 @@ in
         ];
         # Paths only: `argv` is world-readable via /proc/<pid>/cmdline.
         LoadCredential = lib.mapAttrsToList (id: path: "${id}:${path}") logsAuthStoreFiles;
-        DynamicUser = true;
+        # ⚠️ A static user, never `DynamicUser`: with `RuntimeDirectoryPreserve`
+        # systemd puts a dynamic user's runtime directory under
+        # `/run/private/` (0700 root:root), which nginx cannot traverse.
+        User = logsAuthUnit;
+        Group = logsAuthUnit;
         RuntimeDirectory = logsAuthUnit;
         # Traverse-only for others, so nginx reaches the socket path.
         RuntimeDirectoryMode = "0751";
@@ -406,6 +417,8 @@ in
         PrivateTmp = true;
         PrivateDevices = true;
         NoNewPrivileges = true;
+        RestrictSUIDSGID = true;
+        RemoveIPC = true;
       };
       environment = {
         BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";

nix/module-eval/core-toggle.nix

@@ -578,6 +578,47 @@ let
         && lib.hasInfix "--socket /run/swarm-logs-auth/auth.sock" unit.serviceConfig.ExecStart
         && l ? "= /__hive_authelia";
     }
+    {
+      # A `DynamicUser` unit's preserved runtime directory lives under
+      # `/run/private/` (0700 root:root), which nginx cannot traverse, so a
+      # socket there answers every proxied request with a 502. Each `unix:`
+      # upstream under `/run/<dir>/` is matched to the units creating `<dir>`.
+      # The first clause is the control: the match finds swarm-logs-auth's.
+      name = "no unix socket nginx proxies to sits in a DynamicUser's preserved runtime directory";
+      ok =
+        let
+          prefix = "http://unix:/run/";
+          upstreamDirs = lib.unique (
+            map (p: lib.head (lib.splitString "/" (lib.removePrefix prefix p))) (
+              lib.filter (p: p != null && lib.hasPrefix prefix p) (
+                lib.concatMap (vh: lib.mapAttrsToList (_: loc: loc.proxyPass) vh.locations) (
+                  lib.attrValues allLocal.services.nginx.virtualHosts
+                )
+              )
+            )
+          );
+          runtimeDirs =
+            sc:
+            let
+              r = sc.RuntimeDirectory or [ ];
+            in
+            if lib.isList r then r else lib.splitString " " r;
+          owners = lib.filterAttrs (
+            _: u: lib.any (d: lib.elem d upstreamDirs) (runtimeDirs u.serviceConfig)
+          ) allLocal.systemd.services;
+          hidden =
+            sc:
+            lib.elem (sc.DynamicUser or false) [
+              true
+              "yes"
+            ]
+            && !(lib.elem (sc.RuntimeDirectoryPreserve or "no") [
+              false
+              "no"
+            ]);
+        in
+        owners ? swarm-logs-auth && !(lib.any (u: hidden u.serviceConfig) (lib.attrValues owners));
+    }
     {
       # Every agent bearer the responder has not recently granted costs a
       # store login and read, so its location carries the per-address limit.