check every container's root before its start, through the container@ template

nix/host-modules/container-rootfs-check.nix

@@ -0,0 +1,44 @@
+# The pre-start check of a container's root, for every `container@` instance.
+#
+# nixpkgs' start script runs as host root against the container's own root,
+# which is persistent and writable by the container, and its `mkdir`, `chmod`,
+# `touch` and `cp` follow links the container planted there. A drop-in on the
+# `container@` template runs `hive-priv check-rootfs %i` as an `ExecStartPre=`
+# before the script (`hive-priv/src/prestart.rs` has the rules). It covers the
+# agents' imperative instances and the declarative ones (hive-forge,
+# hive-matrix, the swarm services), each of which has a unit file of its own:
+# systemd reads the template's drop-in directory for every instance.
+#
+# Order: systemd parses a drop-in after the unit file, so this line follows the
+# unit file's own `ExecStartPre=` (nixpkgs' `preStart`, which touches no path
+# under the root), and `ExecStart=` runs only after every `ExecStartPre=`
+# succeeded.
+#
+# The drop-in is a unit-file package rather than `systemd.services."container@"`
+# because that attribute renders the template's own unit file, which a
+# declarative instance's unit file replaces.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  dropin = pkgs.writeTextFile {
+    name = "hyperhive-container-rootfs-check";
+    destination = "/lib/systemd/system/container@.service.d/10-hyperhive-rootfs-check.conf";
+    text = ''
+      [Service]
+      ExecStartPre=${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs %i
+    '';
+  };
+in
+{
+  config = lib.mkIf config.boot.enableContainers {
+    systemd.packages = [ dropin ];
+
+    # The per-agent drop-ins this check replaced rooted a hive-priv build per
+    # agent here. Boot is the one time no such drop-in is loaded.
+    systemd.tmpfiles.rules = [ "R! /nix/var/nix/gcroots/hyperhive-check-rootfs - - - - -" ];
+  };
+}

nix/host-modules/default.nix

@@ -22,6 +22,7 @@
     ./hive-matrix-service.nix
     ./hive-matrix.nix
     ./hive-network.nix
+    ./container-rootfs-check.nix
     ./hive-priv.nix
     ./hive-priv-cache.nix
     ./hive-squid.nix

nix/module-eval/core-toggle.nix

@@ -1497,6 +1497,37 @@ let
         && (sc.ProtectKernelLogs or false)
         && (sc.ProtectKernelModules or false);
     }
+    {
+      # Every `container@` instance starts through the template's drop-in, the
+      # agents' and the swarm services' alike, so a host with containers has it
+      # whether or not it runs a hive. A host with none has no `container@`
+      # unit and gets no drop-in. Only the rendered line is pinned: eval cannot
+      # show where systemd reads the file from. The misspelled name is the
+      # control that the lookup finds nothing when nothing is there.
+      name = "the container@ template drop-in runs the root check on every host with containers and on no other";
+      ok =
+        let
+          dropinText =
+            name: m:
+            let
+              hits = lib.filter (p: (p.name or "") == name) m.systemd.packages;
+            in
+            if builtins.length hits == 1 then (builtins.head hits).text else "";
+          runsCheck =
+            name: m:
+            let
+              want = "ExecStartPre=${m.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs %i";
+              lines = lib.splitString "\n" (dropinText name m);
+            in
+            lib.elem (builtins.unsafeDiscardStringContext want) (map builtins.unsafeDiscardStringContext lines);
+          check = "hyperhive-container-rootfs-check";
+        in
+        runsCheck check allLocal
+        && runsCheck check shared.servicesHere
+        && !(runsCheck "${check}-misspelled" allLocal)
+        && !(bare.systemd.units ? "container@.service")
+        && !(runsCheck check bare);
+    }
   ]
   # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
   # at one fires and fetches nothing.

nix/vm-tests/agent-ephemeral-rootfs.nix

@@ -13,10 +13,10 @@
 #     forwarder (nix/agent-modules/otel.nix) reads it, and the host can read it;
 #   - a package installed into the user's nix profile is still there, and on
 #     PATH, after a restart;
-#   - a link planted at `root` in the template refuses the next start through
-#     the limits drop-in's `ExecStartPre=` check (whose build has a GC root),
-#     leaves the link's target untouched, and the start goes through once the
-#     link is gone.
+#   - a link planted at `root` in the root template refuses the next start through
+#     the `ExecStartPre=` check in the `container@` template drop-in
+#     (nix/host-modules/container-rootfs-check.nix), leaves the link's target
+#     untouched, and the start goes through once the link is gone.
 #
 # The agent's journal forwarder itself (an OTLP push to the hive's collector) is
 # not booted here; the module-eval suite covers its wiring.
@@ -69,6 +69,7 @@ pkgs.testers.runNixOSTest {
     {
       imports = [
         hiveStubs
+        ../host-modules/container-rootfs-check.nix
         ../host-modules/hive-priv.nix
       ];
       virtualisation.memorySize = 2048;
@@ -233,21 +234,10 @@ pkgs.testers.runNixOSTest {
 
     with subtest("a link planted in the root refuses the start and nothing is touched"):
         stop()
-        # What hive-c0re does besides WriteNspawnFlags: the limits drop-in,
-        # which carries the pre-start check, and the runtime dir its condition
-        # names.
-        host.succeed(f"mkdir -p /run/hyperhive/agents/{AGENT}")
-        for req in [
-            {"op": "write_resource_limits", "container": MACHINE, "memory_max": "1G", "cpu_quota": "200%"},
-            {"op": "daemon_reload"},
-        ]:
-            out = host.succeed(f"echo '{json.dumps(req)}' | socat - UNIX-CONNECT:/run/hive/priv.sock")
-            assert json.loads(out.strip().splitlines()[-1])["ok"] is True, out
-        dropin = host.succeed(f"cat /run/systemd/system/container@{MACHINE}.service.d/hyperhive-limits.conf")
-        assert "ExecStartPre=/nix/store/" in dropin and f"check-rootfs {MACHINE}\n" in dropin, dropin
-        # The build the drop-in names is rooted against garbage collection.
-        build = "/".join(dropin.split("ExecStartPre=")[1].split()[0].split("/")[:4])
-        assert host.succeed(f"readlink /nix/var/nix/gcroots/hyperhive-check-rootfs/{MACHINE}").strip() == build, build
+        # The pre-start check is a drop-in of the `container@` template, so the
+        # unit carries it with no per-container drop-in written.
+        unit = host.succeed(f"systemctl cat container@{MACHINE}")
+        assert "bin/hive-priv check-rootfs %i\n" in unit, unit
 
         # `root` is a link to a directory outside the root, which the start
         # script would `chmod 0700` as host root.