swarm-victoriametrics: require 1.153.0, carry the history off the DynamicUser path

The socket scheme needs VictoriaMetrics 1.153.0 or later: older builds
create the socket 0600 whatever the umask, so nginx could not connect
and every consumer would get 502 with the module evaluating clean.
deploy.victoriametrics.package is an operator option, so an assertion
now refuses an older one, naming the option and the reason.

Turning DynamicUser off moves the state directory from
/var/lib/private/victoriametrics (behind a /var/lib/victoriametrics
symlink) to a plain /var/lib/victoriametrics. A oneshot ordered before
the store moves the old directory there and chowns it to the static
user when the plain path is the old symlink, empty, or missing, and does
nothing once the private directory is gone. Without it the persistent
container's history depends on systemd's own DynamicUser-off migration.

Refs #4815

nix/host-modules/swarm-victoriametrics.nix

@@ -114,6 +114,22 @@ in
   };
 
   config = lib.mkIf deployCfg.victoriametrics.enable {
+    assertions = [
+      {
+        assertion = lib.versionAtLeast (lib.getVersion deployCfg.victoriametrics.package) "1.153.0";
+        message = ''
+          services.hyperhive.deploy.victoriametrics.package must be
+          VictoriaMetrics 1.153.0 or later (this one is
+          ${lib.getVersion deployCfg.victoriametrics.package}).
+
+          The gateway reaches the store over a unix socket, and only from
+          1.153.0 does VictoriaMetrics create that socket under the process
+          umask. An older build creates it 0600 whatever the umask, so nginx
+          cannot connect and every read, push and scrape fails with 502.
+        '';
+      }
+    ];
+
     # The gateway name, inside `deployCfg.victoriametrics.enable` — that
     # guard is the load-bearing part. Every hive in a swarm may know this
     # store exists, but only the host that RUNS it may claim the name; a
@@ -224,7 +240,7 @@ in
       };
 
       config =
-        { pkgs, ... }:
+        { config, pkgs, ... }:
         {
           imports = [ ../container-modules/swarm-container.nix ];
 
@@ -262,6 +278,49 @@ in
             '';
           };
 
+          # Moves the store's history out of the DynamicUser layout, where
+          # `/var/lib/<stateDir>` is a symlink into `/var/lib/private/`, to the
+          # plain directory the static user's StateDirectory is. Runs before
+          # the store on every start and does nothing once
+          # `/var/lib/private/<stateDir>` is gone. When both paths hold data
+          # it moves neither and the store starts on the plain one.
+          systemd.services.victoriametrics-state-migrate =
+            let
+              old = "/var/lib/private/${config.services.victoriametrics.stateDir}";
+              new = "/var/lib/${config.services.victoriametrics.stateDir}";
+            in
+            {
+              description = "move VictoriaMetrics' DynamicUser state directory to its static path";
+              before = [ "victoriametrics.service" ];
+              requiredBy = [ "victoriametrics.service" ];
+              serviceConfig = {
+                Type = "oneshot";
+                RemainAfterExit = true;
+                SyslogIdentifier = "victoriametrics-state-migrate";
+              };
+              path = [ pkgs.coreutils ];
+              script = ''
+                set -eu
+                old=${lib.escapeShellArg old}
+                new=${lib.escapeShellArg new}
+                if [ ! -d "$old" ] || [ -L "$old" ]; then
+                  exit 0
+                fi
+                if [ -L "$new" ]; then
+                  rm "$new"
+                elif [ -d "$new" ]; then
+                  if [ -n "$(ls -A "$new")" ]; then
+                    echo "victoriametrics-state-migrate: $old and $new both hold data; moving neither" >&2
+                    exit 0
+                  fi
+                  rmdir "$new"
+                fi
+                mv "$old" "$new"
+                chown -R victoriametrics:gateway-nginx "$new"
+                echo "victoriametrics-state-migrate: moved $old to $new"
+              '';
+            };
+
           services.victoriametrics = {
             enable = true;
             package = deployCfg.victoriametrics.package;

nix/module-eval/swarm-services-switch.nix

@@ -169,6 +169,23 @@ let
     deploy.ci-runner.enable = true;
   };
   serviceContainers = serviceContainersWithCi.containers;
+
+  # The metrics store on a build older than its socket needs. Only its
+  # `assertions` are read, and the package is a stub carrying a version.
+  oldMetricsStore = hive {
+    deploy.victoriametrics.enable = true;
+    deploy.victoriametrics.package = pkgs.emptyDirectory // {
+      version = "1.152.0";
+    };
+  };
+  # Failing assertions first, then their messages: a message is only
+  # well-defined when its assertion fails (nixpkgs' fileSystems one throws
+  # otherwise).
+  failedMetricsStoreVersion =
+    host:
+    lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
+      map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
+    );
   cases = [
     {
       # `lib.all` over an empty set holds vacuously, so the roster is counted
@@ -537,6 +554,35 @@ let
         && unit.serviceConfig.UMask == "0007"
         && lib.hasInfix "--unix-socket ${sock}" unit.postStart;
     }
+    {
+      # The DynamicUser layout keeps the history under /var/lib/private; the
+      # static user's StateDirectory is the plain path. The move runs as its
+      # own unit ahead of the store.
+      name = "the metrics store's history moves from the DynamicUser path before the store starts";
+      ok =
+        let
+          cfg = serviceContainers.swarm-victoriametrics.config;
+          m = cfg.systemd.services.victoriametrics-state-migrate;
+        in
+        cfg.services.victoriametrics.stateDir == "victoriametrics"
+        && builtins.elem "victoriametrics.service" m.before
+        && builtins.elem "victoriametrics.service" m.requiredBy
+        && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
+        && lib.hasInfix "new=/var/lib/victoriametrics" m.script
+        && lib.hasInfix ''mv "$old" "$new"'' m.script
+        && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
+    }
+    {
+      # The control is the default package on the all-services host.
+      name = "a metrics store package older than 1.153.0 fails evaluation";
+      ok =
+        let
+          old = failedMetricsStoreVersion oldMetricsStore;
+        in
+        lib.length old == 1
+        && lib.hasInfix "1.152.0" (lib.head old)
+        && failedMetricsStoreVersion serviceContainersWithCi == [ ];
+    }
     {
       name = "a service container that sets no stateVersion of its own is on 26.05";
       ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";