diff --git a/nix/host-modules/hive-ci.nix b/nix/host-modules/hive-ci.nix index 003d175f..dcce7244 100644 --- a/nix/host-modules/hive-ci.nix +++ b/nix/host-modules/hive-ci.nix @@ -202,6 +202,17 @@ in hostPath = "/run/hive-ci/runner-token"; isReadOnly = true; }; + # Route the container's nix through the HOST nix-daemon (as the agent + # containers do) instead of an in-container daemon: with + # NIX_REMOTE=daemon below, builds run on the host daemon and inherit + # its buildMachines + max-jobs + fallback, so a remote builder that's + # down degrades to a local build instead of hard-failing the check. + # Bind the *directory* (not the socket file) so the mount stays live + # across a host nix-daemon restart, which recreates the socket inode. + "/nix/var/nix/daemon-socket" = { + hostPath = "/nix/var/nix/daemon-socket"; + isReadOnly = false; + }; } # Self-signed mode: bind the public hive CA cert read-only so the # runner's Node actions trust the gateway/forge leaf (consumed via @@ -246,6 +257,16 @@ in "flakes" ]; + # Build through the HOST nix-daemon (bind-mounted socket above), not + # an in-container daemon. NIX_REMOTE=daemon makes root services (the + # runner) use the daemon socket rather than store=auto → local store. + # Disable BOTH the service and its .socket unit: the .socket owns the + # listen path /nix/var/nix/daemon-socket/socket and would otherwise + # race the bind mount over the same path. + systemd.globalEnvironment.NIX_REMOTE = "daemon"; + systemd.services.nix-daemon.enable = false; + systemd.sockets.nix-daemon.enable = false; + # package is top-level on gitea-actions-runner, not per-instance. services.gitea-actions-runner.package = cfg.package; @@ -310,37 +331,19 @@ in environment = lib.mkIf useSelfSigned { NODE_EXTRA_CA_CERTS = caContainerPath; }; - # Gate runner start (and therefore job registration/claiming) on - # the in-container nix daemon being reachable. After a hive-ci - # restart the runner re-registers and immediately claims any - # queued jobs — which can beat the nix daemon coming up: its - # `nix-daemon.socket` carries - # `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is - # skipped until /nix/var is read-write, so the first - # nix-dependent build dispatches into a cold daemon and - # hangs/retries (observed: a 55m48s `nix flake check` vs the - # normal ~30s, a build-offload stall, not a code failure). + # Gate runner start on credentials being present. Registration is + # done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a + # token and, via hive-priv, writes /run/hive-ci/runner-token + + # restarts this unit. Pass iff already registered (.runner present) + # OR a real (non-placeholder) token is in place; else exit 1 so the + # runner stays down (Restart=on-failure retries) until c0re + # registers it, rather than blocking the container's start. # - # Ordering `after`/`wants` the socket unit does NOT fix this — a - # condition-skipped unit satisfies systemd ordering immediately, - # so the runner would still start before the daemon is live. - # Instead block in ExecStartPre by polling the daemon until it - # actually answers; this is topology-agnostic (works whether the - # daemon is in-container or a shared host socket). `mkBefore` so - # this runs ahead of any pre-steps the upstream module adds. + # No nix-daemon wait anymore: builds route through the HOST daemon + # (bound socket above), which is up before the container — nix's + # own connect retries cover any brief window. `mkBefore` runs this + # ahead of any upstream pre-steps. serviceConfig.ExecStartPre = lib.mkBefore [ - # Fail fast unless the runner can actually come up. Registration - # is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints - # a token and, via hive-priv, writes /run/hive-ci/runner-token + - # restarts this unit — so the runner never does the forge - # round-trip on the boot path (that was the old prefetch's - # boot-critical wait that could trip the nspawn start timeout). - # Pass iff already registered (.runner present) OR a real - # (non-placeholder) token is in place; otherwise exit 1 so the - # runner just stays down (Restart=on-failure retries) until c0re - # registers it, instead of blocking the container's start. Runs - # ahead of the nix-daemon wait below — no point waiting for the - # daemon if there are no credentials to come up with. (pkgs.writeShellScript "hive-ci-runner-precond" '' RUNNER=/var/lib/gitea-runner/hive/.runner TOKEN_FILE=/run/hive-ci/runner-token @@ -351,36 +354,6 @@ in echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2 exit 1 '') - (pkgs.writeShellScript "wait-nix-daemon" '' - # Up to ~180s; the daemon is normally up within seconds, this - # only bites in the post-restart cold window. Non-fatal shape: - # if it never comes up the unit fails cleanly (recoverable) - # rather than the runner claiming jobs into a dead daemon. - # - # `--store daemon` is load-bearing. A bare `nix store ping` - # uses the `auto` store, which — when run as root with the - # daemon socket absent — silently resolves to a LOCAL store - # (root can write /nix/store directly) and pings successfully. - # systemd service units don't source the profile that sets - # `NIX_REMOTE=daemon`, so this is the real environment here. - # At cold boot the daemon socket IS absent: nix-daemon.socket - # carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` - # and is condition-skipped until /nix/var goes read-write. So - # a bare ping would pass against the local store while the - # daemon is still down — defeating the gate's whole purpose - # (the runner would start and claim jobs the daemon can't yet - # service). Pinning `--store daemon` makes the poll verify the - # actual daemon socket, so the gate honours its contract and - # waits until the daemon — not a local fallback — answers. - for _ in $(seq 1 90); do - if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then - exit 0 - fi - sleep 2 - done - echo "nix daemon not reachable after 180s" >&2 - exit 1 - '') ]; # Registration is out of band: hive-c0re explicitly restarts this # unit once it writes the real token, which is the primary path. As