swarm-victoriametrics: require 1.153.0, carry the history off the DynamicUser path

The socket scheme needs VictoriaMetrics 1.153.0 or later: older builds
create the socket 0600 whatever the umask, so nginx could not connect
and every consumer would get 502 with the module evaluating clean.
deploy.victoriametrics.package is an operator option, so an assertion
now refuses an older one, naming the option and the reason.

Turning DynamicUser off moves the state directory from
/var/lib/private/victoriametrics (behind a /var/lib/victoriametrics
symlink) to a plain /var/lib/victoriametrics. A oneshot ordered before
the store moves the old directory there and chowns it to the static
user when the plain path is the old symlink, empty, or missing, and does
nothing once the private directory is gone. Without it the persistent
container's history depends on systemd's own DynamicUser-off migration.

Refs #4815

docs/swarm/services.md

@@ -168,8 +168,10 @@ Grafana start — see [`secrets.md`](secrets.md).
 
 **Where the data comes from.** The swarm's OTEL collector, below.
 
-Neither container is reachable except through the gateway: both bind
+Neither container is reachable except through the gateway: each listens
-loopback, and VictoriaMetrics authenticates nothing itself. The
+on a unix socket only the gateway's nginx can open, and VictoriaMetrics
+runs in a network namespace of its own with no interface but loopback.
+VictoriaMetrics authenticates nothing itself. The
 `metrics.<swarm-domain>` vhost forwards only the store's read API (the
 Prometheus query, series, label and metadata routes, plus the vmui web UI)
 without a credential. It forwards the collector's OTLP push, and the
@@ -206,9 +208,8 @@ swarm.
 | `swarm.victorialogs.domain`           | Only to rename it.                                                                                                                                                                                                                     |
 | `swarm.victorialogs.port`             | Only if something else on the services host already claims `9428`.                                                                                                                                                                     |
 
-Like the metrics store, it binds loopback and takes no credential of its
+It binds loopback and takes no credential of its own: the gateway vhost is
-own: the gateway vhost is the only way in, and the collector is the only
+the only way in, and the collector is the only intended writer.
-intended writer.
 
 The collector also scrapes this store's own metrics, at
 `https://<swarm.victorialogs.domain>/metrics`, so a collector on a host

nix/host-modules/deploy.nix

@@ -80,6 +80,11 @@ in
       [ "services" "hyperhive" "swarm" "victoriametrics" "enable" ]
       [ "services" "hyperhive" "deploy" "victoriametrics" "enable" ]
     )
+    (lib.mkRemovedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "port" ] ''
+      VictoriaMetrics listens on a unix socket in a network namespace of its
+      own, and the gateway reaches it there, so it binds no host port. Remove
+      this definition.
+    '')
     (lib.mkRenamedOptionModule
       [ "services" "hyperhive" "swarm" "victorialogs" "enable" ]
       [ "services" "hyperhive" "deploy" "victorialogs" "enable" ]

nix/host-modules/swarm-otel.nix

@@ -757,8 +757,7 @@ in
               cfg.producerPort
               cfg.storeProducerPort
               otelCfg.collector.port
-            ]
+            ];
-            ++ lib.optional deployCfg.victoriametrics.enable vmCfg.port;
             all = derived ++ others;
           in
           lib.length (lib.unique all) == lib.length all;

nix/host-modules/swarm-victorialogs.nix

@@ -455,8 +455,7 @@ in
             enable = true;
             package = deployCfg.victorialogs.package;
 
-            # ⚠️ PINNED TO LOOPBACK for the same reason the metrics store is,
+            # ⚠️ PINNED TO LOOPBACK: upstream's default listens on every
-            # and it matters more here: upstream's default listens on every
             # interface, and this endpoint accepts writes as well as reads.
             # Nothing authenticates it either — upstream offers one basic-auth
             # pair (`-httpAuth.username` / `-httpAuth.password`) and this

nix/host-modules/swarm-victoriametrics-service.nix

@@ -1,5 +1,5 @@
-# The swarm's metrics store as every hive sees it: the name it answers on and
+# The swarm's metrics store as every hive sees it: the name it answers on,
-# its port, identical on every host. What the host running it decides, and the
+# identical on every host. What the host running it decides, and the
 # container itself, are in ./swarm-victoriametrics.nix.
 {
   lib,
@@ -16,8 +16,8 @@ let
   domainBase = if swarmDomain == null then "invalid" else swarmDomain;
 in
 {
-  # What the store IS from any hive's point of view: the name it answers on and
+  # What the store IS from any hive's point of view: the name it answers on.
-  # the port. `enable`, `package` and `retentionPeriod` are decisions of the
+  # `enable`, `package` and `retentionPeriod` are decisions of the
   # host that runs it and live under `deploy.*`.
   options.services.hyperhive.swarm.victoriametrics = {
     machine = lib.mkOption {
@@ -42,18 +42,5 @@ in
         child decides which CA may sign it.
       '';
     };
-
-    port = lib.mkOption {
-      type = lib.types.port;
-      default = 8428;
-      description = ''
-        Port VictoriaMetrics listens on, bound to loopback only on the
-        host that runs it: its write endpoint is unauthenticated, so the
-        gateway's vhost there is the only route to it. Upstream's own
-        default, kept so an operator reading VictoriaMetrics
-        documentation finds what they expect.
-      '';
-    };
-
   };
 }

nix/host-modules/swarm-victoriametrics.nix

@@ -18,13 +18,23 @@
 let
   cfg = config.services.hyperhive.swarm.victoriametrics;
   deployCfg = config.services.hyperhive.deploy;
-  networkCfg = config.services.hyperhive.network;
   hyperhiveCfg = config.services.hyperhive;
   gatewayCfg = hyperhiveCfg.gateway;
 
-  # Shared host netns, like every sibling swarm container: the gateway
+  # A netns of its own with no interface but loopback: the store makes no
-  # reaches this at 127.0.0.1:<port>.
+  # outbound calls, and the gateway reaches it over `socketPath` below, so
-  privateNetwork = false;
+  # nothing on the host or the hive bridge can dial its listener.
+  privateNetwork = true;
+
+  # Fixed, not an option: the directory's mode is what keeps the socket
+  # private, so it holds this one socket and nothing else.
+  socketDir = "/run/swarm-victoriametrics";
+  socketPath = "${socketDir}/victoriametrics.sock";
+  upstream = path: "http://unix:${socketPath}:${path}";
+
+  # The gid the gateway's nginx runs as. Only the number crosses into the
+  # container, which knows it as `gateway-nginx`.
+  nginxGid = config.ids.gids.nginx;
 
   # The store's read API: the only part of it the vhost serves without a
   # credential. Each location's `proxy_pass` carries the path, so the store
@@ -64,7 +74,7 @@ let
   ];
   readLocations =
     let
-      to = path: { proxyPass = "http://127.0.0.1:${toString cfg.port}${path}"; };
+      to = path: { proxyPass = upstream path; };
     in
     lib.listToAttrs (
       map (p: lib.nameValuePair "= ${p}" (to p)) readPaths
@@ -72,8 +82,8 @@ let
     );
 in
 {
-  # What the store IS from any hive's point of view, the name it answers on and
+  # What the store IS from any hive's point of view, the name it answers on,
-  # the port, is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
+  # is `swarm.victoriametrics` in ./swarm-victoriametrics-service.nix.
   # `enable`, `package` and `retentionPeriod` are decisions of the host that
   # runs it and live under `deploy.*`.
 
@@ -104,6 +114,22 @@ in
   };
 
   config = lib.mkIf deployCfg.victoriametrics.enable {
+    assertions = [
+      {
+        assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
+        message = ''
+          services.hyperhive.deploy.victoriametrics.package must be
+          VictoriaMetrics 1.153.0 or later (this one is
+          ${lib.getVersion deployCfg.victoriametrics.package}).
+
+          The gateway reaches the store over a unix socket, and only from
+          1.153.0 does VictoriaMetrics create that socket under the process
+          umask. An older build creates it 0600 whatever the umask, so nginx
+          cannot connect and every read, push and scrape fails with 502.
+        '';
+      }
+    ];
+
     # The gateway name, inside `deployCfg.victoriametrics.enable` — that
     # guard is the load-bearing part. Every hive in a swarm may know this
     # store exists, but only the host that RUNS it may claim the name; a
@@ -111,8 +137,7 @@ in
     # have.
     services.hyperhive.gateway.localNames = [ cfg.domain ];
 
-    # This host serves the vhost, and the container behind it resolves
+    # This host serves the vhost, and the hive's dnsmasq answers its name.
-    # through the hive's dnsmasq.
     services.hyperhive.gateway.enable = lib.mkDefault true;
     services.hyperhive.gateway.dns.enable = lib.mkDefault true;
 
@@ -138,7 +163,7 @@ in
         # follow it and POST its batch at a login page that answers 200 —
         # ingest reporting healthy while storing nothing.
         "= /opentelemetry/api/v1/push" = {
-          proxyPass = "http://127.0.0.1:${toString cfg.port}/opentelemetry/api/v1/push";
+          proxyPass = upstream "/opentelemetry/api/v1/push";
           extraConfig = ''
             auth_request /__metrics_push_authz;
           '';
@@ -153,7 +178,7 @@ in
         # `publishedScrapeTargets.victoriametrics` in ./swarm-otel-service.nix;
         # the two must agree, since that string is also the token's audience.
         "= /metrics" = {
-          proxyPass = "http://127.0.0.1:${toString cfg.port}/metrics";
+          proxyPass = upstream "/metrics";
           extraConfig = ''
             auth_request /__metrics_push_authz;
           '';
@@ -188,6 +213,17 @@ in
       };
     };
 
+    # The socket directory, created host-side before the container starts:
+    # nixos-container refuses to start when a bind source is missing.
+    #
+    # Root-owned and group nginx, `0770`: the store creates its socket here
+    # through the group (its uid is allocated inside the container, so the
+    # host cannot name it), and the gateway traverses it the same way.
+    # Everything else is kept out.
+    systemd.tmpfiles.rules = [
+      "d ${socketDir} 0770 root ${toString nginxGid} - -"
+    ];
+
     containers.${cfg.machine} = {
       autoStart = true;
       ephemeral = false;
@@ -196,31 +232,105 @@ in
       extraFlags = [ "--link-journal=host" ];
       inherit privateNetwork;
 
-      config =
+      # A bind mount rather than a path both sides happen to know: a
-        { ... }:
+      # container's `/run` is an nspawn tmpfs the host cannot see into.
-        {
+      bindMounts.${socketDir} = {
-          imports = [
+        hostPath = socketDir;
-            ../container-modules/swarm-container.nix
+        isReadOnly = false;
-            (import ../container-modules/swarm-container-resolver.nix {
+      };
-              inherit (networkCfg) bridgeIp;
-              dnsConsumers = [ "victoriametrics.service" ];
-            })
-          ];
 
-          services.hyperhive.swarmContainer = { inherit privateNetwork; };
+      config =
+        { config, pkgs, ... }:
+        {
+          imports = [ ../container-modules/swarm-container.nix ];
+
+          # No resolver unit: with no interface but loopback there is no
+          # nameserver to reach, and the store resolves no names.
+          services.hyperhive.swarmContainer = {
+            inherit privateNetwork;
+            writesOwnResolvConf = false;
+          };
+
+          # The socket's group is the store's primary group, so that group is
+          # nginx's gid. A static user, not nixpkgs' DynamicUser: a dynamic
+          # user's group is allocated per start, and nginx is never in it.
+          users.groups.gateway-nginx.gid = nginxGid;
+          users.users.victoriametrics = {
+            isSystemUser = true;
+            group = "gateway-nginx";
+          };
+
+          systemd.services.victoriametrics = {
+            serviceConfig = {
+              DynamicUser = lib.mkForce false;
+              User = "victoriametrics";
+              Group = "gateway-nginx";
+              # The store creates its socket under the process umask, so
+              # `0007` makes it `0770` and nginx's group can connect.
+              UMask = "0007";
+            };
+            # nixpkgs' readiness loop curls `http://<listenAddress>/ping`,
+            # which no `unix:` address answers.
+            postStart = lib.mkForce ''
+              until ${lib.getExe pkgs.curl} -s -o /dev/null --unix-socket ${socketPath} http://localhost/ping; do
+                sleep 1;
+              done
+            '';
+          };
+
+          # Moves the store's history out of the DynamicUser layout, where
+          # `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
+          # plain directory the static user's StateDirectory is. Runs before
+          # the store on every start and does nothing once
+          # `/var/lib/private/<stateDir>` is gone. When both paths hold data
+          # it moves neither and the store starts on the plain one.
+          systemd.services.victoriametrics-state-migrate =
+            let
+              old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
+              new = "/var/lib/${config.services.victoriametrics.stateDir}";
+            in
+            {
+              description = "move VictoriaMetrics' DynamicUser state directory to its static path";
+              before = [ "victoriametrics.service" ];
+              requiredBy = [ "victoriametrics.service" ];
+              serviceConfig = {
+                Type = "oneshot";
+                RemainAfterExit = true;
+                SyslogIdentifier = "victoriametrics-state-migrate";
+              };
+              path = [ pkgs.coreutils ];
+              script = ''
+                set -eu
+                old=${lib.escapeShellArg old}
+                new=${lib.escapeShellArg new}
+                if [ ! -d "$old" ] || [ -L "$old" ]; then
+                  exit 0
+                fi
+                if [ -L "$new" ]; then
+                  rm "$new"
+                elif [ -d "$new" ]; then
+                  if [ -n "$(ls -A "$new")" ]; then
+                    echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
+                    exit 0
+                  fi
+                  rmdir "$new"
+                fi
+                mv "$old" "$new"
+                chown -R victoriametrics:gateway-nginx "$new"
+                echo "victoriametrics-state-migrate: moved $old to $new"
+              '';
+            };
 
           services.victoriametrics = {
             enable = true;
             package = deployCfg.victoriametrics.package;
             retentionPeriod = deployCfg.victoriametrics.retentionPeriod;
 
-            # ⚠️ PINNED TO LOOPBACK, and this is a correction rather than a
+            # ⚠️ A socket, not upstream's default `:8428` on every interface:
-            # preference: upstream's default is `:8428`, i.e. every
+            # the gateway is the only intended client, and this listener
-            # interface. The gateway is the only intended client and it is on
+            # carries an unauthenticated write endpoint (see the OTLP note
-            # this host, so binding wider would publish an unauthenticated
+            # below).
-            # write endpoint (see the OTLP note below) to whatever the host
+            listenAddress = "unix:${socketPath}";
-            # is reachable on.
-            listenAddress = "127.0.0.1:${toString cfg.port}";
           };
         };
     };
@@ -235,8 +345,8 @@ in
   # deployment ever needs them.
   #
   # ⚠️ That endpoint is unauthenticated, which is why `listenAddress` above is
-  # loopback: the only route to it from off-host is the vhost's ingest
+  # a socket only the gateway can reach: the only route to it is the vhost's
-  # location, which authenticates. The collector is still the only writer, but
+  # ingest location, which authenticates. The collector is still the only writer, but
   # it now arrives by the swarm name rather than over loopback, because it need
   # not share a host with this store.
 }

nix/module-eval/swarm-otel-identity.nix

@@ -282,7 +282,7 @@ let
           m = locs."= /metrics" or null;
         in
         m != null
-        && m.proxyPass == "http://127.0.0.1:${toString h.swarm.victoriametrics.port}/metrics"
+        && m.proxyPass == "http://unix:/run/swarm-victoriametrics/victoriametrics.sock:/metrics"
         && lib.hasInfix "auth_request /__metrics_push_authz;" m.extraConfig
         && !(lib.hasInfix "error_page" m.extraConfig)
         && lib.hasInfix "internal;" locs."= /__metrics_push_authz".extraConfig

nix/module-eval/swarm-services-switch.nix

@@ -161,14 +161,31 @@ let
     lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;
 
   # Every service container at once: the services-only host plus the CI
-  # runner. The CI runner and the review classifier are the two containers on
+  # runner. The CI runner, the review classifier and the metrics store are the
-  # a netns of their own.
+  # three containers on a netns of their own.
   serviceContainersWithCi = hive {
     deploy.hive-controller.enable = false;
     deploy.allSwarmServices = true;
     deploy.ci-runner.enable = true;
   };
   serviceContainers = serviceContainersWithCi.containers;
+
+  # The metrics store on a build older than its socket needs. Only its
+  # `assertions` are read, and the package is a stub carrying a version.
+  oldMetricsStore = hive {
+    deploy.victoriametrics.enable = true;
+    deploy.victoriametrics.package = pkgs.emptyDirectory // {
+      version = "1.152.0";
+    };
+  };
+  # Failing assertions first, then their messages: a message is only
+  # well-defined when its assertion fails (nixpkgs' fileSystems one throws
+  # otherwise).
+  failedMetricsStoreVersion =
+    host:
+    lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
+      map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
+    );
   cases = [
     {
       # `lib.all` over an empty set holds vacuously, so the roster is counted
@@ -484,7 +501,7 @@ let
         let
           shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
         in
-        lib.length shared == 9
+        lib.length shared == 8
         && lib.all (
           c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
         ) shared;
@@ -499,6 +516,73 @@ let
         in
         c.networking.firewall.enable && c.networking.resolvconf.enable;
     }
+    {
+      # No interface but loopback, so the socket is the store's only listener
+      # the host can reach. Every location that forwards to the store goes
+      # through it; the authelia subrequest is the one `https://` upstream.
+      # The read, push and scrape locations are named so an empty filter
+      # can't pass.
+      name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
+      ok =
+        let
+          host = serviceContainersWithCi;
+          c = host.containers.swarm-victoriametrics;
+          dir = "/run/swarm-victoriametrics";
+          sock = "${dir}/victoriametrics.sock";
+          viaSock = path: "http://unix:${sock}:${path}";
+          locs =
+            host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
+          toStore = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
+            map (l: l.proxyPass) (lib.attrValues locs)
+          );
+          unit = c.config.systemd.services.victoriametrics;
+          nginxGid = host.ids.gids.nginx;
+        in
+        c.privateNetwork
+        && c.hostBridge == null
+        && c.hostAddress == null
+        && c.config.services.victoriametrics.listenAddress == "unix:${sock}"
+        && locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query"
+        && locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push"
+        && locs."= /metrics".proxyPass == viaSock "/metrics"
+        && lib.all (lib.hasPrefix (viaSock "")) toStore
+        && c.bindMounts.${dir}.hostPath == dir
+        && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
+        && c.config.users.groups.gateway-nginx.gid == nginxGid
+        && unit.serviceConfig.Group == "gateway-nginx"
+        && !unit.serviceConfig.DynamicUser
+        && unit.serviceConfig.UMask == "0007"
+        && lib.hasInfix "--unix-socket ${sock}" unit.postStart;
+    }
+    {
+      # The DynamicUser layout keeps the history under /var/lib/private; the
+      # static user's StateDirectory is the plain path. The move runs as its
+      # own unit ahead of the store.
+      name = "the metrics store's history moves from the DynamicUser path before the store starts";
+      ok =
+        let
+          cfg = serviceContainers.swarm-victoriametrics.config;
+          m = cfg.systemd.services.victoriametrics-state-migrate;
+        in
+        cfg.services.victoriametrics.stateDir == "victoriametrics"
+        && builtins.elem "victoriametrics.service" m.before
+        && builtins.elem "victoriametrics.service" m.requiredBy
+        && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
+        && lib.hasInfix "new=/var/lib/victoriametrics" m.script
+        && lib.hasInfix ''mv "$old" "$new"'' m.script
+        && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
+    }
+    {
+      # The control is the default package on the all-services host.
+      name = "a metrics store package older than 1.153.0 fails evaluation";
+      ok =
+        let
+          old = failedMetricsStoreVersion oldMetricsStore;
+        in
+        lib.length old == 1
+        && lib.hasInfix "1.152.0" (lib.head old)
+        && failedMetricsStoreVersion serviceContainersWithCi == [ ];
+    }
     {
       name = "a service container that sets no stateVersion of its own is on 26.05";
       ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";