diff --git a/nix/modules/hive-ci.nix b/nix/modules/hive-ci.nix index b509aad8..abe82b3b 100644 --- a/nix/modules/hive-ci.nix +++ b/nix/modules/hive-ci.nix @@ -406,7 +406,43 @@ in pkgs.nix ]; - systemd.services."gitea-runner-hive".path = [ pkgs.nix ]; + systemd.services."gitea-runner-hive" = { + path = [ pkgs.nix ]; + # Gate runner start (and therefore job registration/claiming) on + # the in-container nix daemon being reachable. After a hive-ci + # restart the runner re-registers and immediately claims any + # queued jobs — which can beat the nix daemon coming up: its + # `nix-daemon.socket` carries + # `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is + # skipped until /nix/var is read-write, so the first + # nix-dependent build dispatches into a cold daemon and + # hangs/retries (observed: a 55m48s `nix flake check` vs the + # normal ~30s, a build-offload stall, not a code failure). + # + # Ordering `after`/`wants` the socket unit does NOT fix this — a + # condition-skipped unit satisfies systemd ordering immediately, + # so the runner would still start before the daemon is live. + # Instead block in ExecStartPre by polling the daemon until it + # actually answers; this is topology-agnostic (works whether the + # daemon is in-container or a shared host socket). `mkBefore` so + # this runs ahead of any pre-steps the upstream module adds. + serviceConfig.ExecStartPre = lib.mkBefore [ + (pkgs.writeShellScript "wait-nix-daemon" '' + # Up to ~180s; the daemon is normally up within seconds, this + # only bites in the post-restart cold window. Non-fatal shape: + # if it never comes up the unit fails cleanly (recoverable) + # rather than the runner claiming jobs into a dead daemon. + for _ in $(seq 1 90); do + if ${pkgs.nix}/bin/nix store ping >/dev/null 2>&1; then + exit 0 + fi + sleep 2 + done + echo "nix daemon not reachable after 180s" >&2 + exit 1 + '') + ]; + }; }; }; };