fix(#2578): route hive-ci's nix through the host daemon (keep distributed builds + gain fallback)
CI's nix flake check ran in hive-ci's OWN in-container nix-daemon, which offloads to the pc2 remote builder and HARD-FAILS when pc2 is unreachable (Connection-reset) — reddening every PR's queue hive-wide. The host daemon builds fine in the same situation (buildMachines + max-jobs>=1 + fallback → local build when pc2 is down), and the agent containers already route through it. Give hive-ci the same wiring: bind-mount the host nix-daemon socket dir into the container, set NIX_REMOTE=daemon, and disable the container's own nix-daemon service + socket. Now CI builds through the host daemon — pc2 offload when it's up, graceful local fallback when it's down. Drops the now-moot in-container wait-nix-daemon precond. Needs an operator rebuild to apply.
This commit is contained in:
atlas
mara
parent
4287435696
commit
6559f3e7b5
1 changed files with 32 additions and 59 deletions
|
|
@ -202,6 +202,17 @@ in
|
|||
hostPath = "/run/hive-ci/runner-token";
|
||||
isReadOnly = true;
|
||||
};
|
||||
# Route the container's nix through the HOST nix-daemon (as the agent
|
||||
# containers do) instead of an in-container daemon: with
|
||||
# NIX_REMOTE=daemon below, builds run on the host daemon and inherit
|
||||
# its buildMachines + max-jobs + fallback, so a remote builder that's
|
||||
# down degrades to a local build instead of hard-failing the check.
|
||||
# Bind the *directory* (not the socket file) so the mount stays live
|
||||
# across a host nix-daemon restart, which recreates the socket inode.
|
||||
"/nix/var/nix/daemon-socket" = {
|
||||
hostPath = "/nix/var/nix/daemon-socket";
|
||||
isReadOnly = false;
|
||||
};
|
||||
}
|
||||
# Self-signed mode: bind the public hive CA cert read-only so the
|
||||
# runner's Node actions trust the gateway/forge leaf (consumed via
|
||||
|
|
@ -246,6 +257,16 @@ in
|
|||
"flakes"
|
||||
];
|
||||
|
||||
# Build through the HOST nix-daemon (bind-mounted socket above), not
|
||||
# an in-container daemon. NIX_REMOTE=daemon makes root services (the
|
||||
# runner) use the daemon socket rather than store=auto → local store.
|
||||
# Disable BOTH the service and its .socket unit: the .socket owns the
|
||||
# listen path /nix/var/nix/daemon-socket/socket and would otherwise
|
||||
# race the bind mount over the same path.
|
||||
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
||||
systemd.services.nix-daemon.enable = false;
|
||||
systemd.sockets.nix-daemon.enable = false;
|
||||
|
||||
# package is top-level on gitea-actions-runner, not per-instance.
|
||||
services.gitea-actions-runner.package = cfg.package;
|
||||
|
||||
|
|
@ -310,37 +331,19 @@ in
|
|||
environment = lib.mkIf useSelfSigned {
|
||||
NODE_EXTRA_CA_CERTS = caContainerPath;
|
||||
};
|
||||
# Gate runner start (and therefore job registration/claiming) on
|
||||
# the in-container nix daemon being reachable. After a hive-ci
|
||||
# restart the runner re-registers and immediately claims any
|
||||
# queued jobs — which can beat the nix daemon coming up: its
|
||||
# `nix-daemon.socket` carries
|
||||
# `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket` and is
|
||||
# skipped until /nix/var is read-write, so the first
|
||||
# nix-dependent build dispatches into a cold daemon and
|
||||
# hangs/retries (observed: a 55m48s `nix flake check` vs the
|
||||
# normal ~30s, a build-offload stall, not a code failure).
|
||||
# Gate runner start on credentials being present. Registration is
|
||||
# done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints a
|
||||
# token and, via hive-priv, writes /run/hive-ci/runner-token +
|
||||
# restarts this unit. Pass iff already registered (.runner present)
|
||||
# OR a real (non-placeholder) token is in place; else exit 1 so the
|
||||
# runner stays down (Restart=on-failure retries) until c0re
|
||||
# registers it, rather than blocking the container's start.
|
||||
#
|
||||
# Ordering `after`/`wants` the socket unit does NOT fix this — a
|
||||
# condition-skipped unit satisfies systemd ordering immediately,
|
||||
# so the runner would still start before the daemon is live.
|
||||
# Instead block in ExecStartPre by polling the daemon until it
|
||||
# actually answers; this is topology-agnostic (works whether the
|
||||
# daemon is in-container or a shared host socket). `mkBefore` so
|
||||
# this runs ahead of any pre-steps the upstream module adds.
|
||||
# No nix-daemon wait anymore: builds route through the HOST daemon
|
||||
# (bound socket above), which is up before the container — nix's
|
||||
# own connect retries cover any brief window. `mkBefore` runs this
|
||||
# ahead of any upstream pre-steps.
|
||||
serviceConfig.ExecStartPre = lib.mkBefore [
|
||||
# Fail fast unless the runner can actually come up. Registration
|
||||
# is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
|
||||
# a token and, via hive-priv, writes /run/hive-ci/runner-token +
|
||||
# restarts this unit — so the runner never does the forge
|
||||
# round-trip on the boot path (that was the old prefetch's
|
||||
# boot-critical wait that could trip the nspawn start timeout).
|
||||
# Pass iff already registered (.runner present) OR a real
|
||||
# (non-placeholder) token is in place; otherwise exit 1 so the
|
||||
# runner just stays down (Restart=on-failure retries) until c0re
|
||||
# registers it, instead of blocking the container's start. Runs
|
||||
# ahead of the nix-daemon wait below — no point waiting for the
|
||||
# daemon if there are no credentials to come up with.
|
||||
(pkgs.writeShellScript "hive-ci-runner-precond" ''
|
||||
RUNNER=/var/lib/gitea-runner/hive/.runner
|
||||
TOKEN_FILE=/run/hive-ci/runner-token
|
||||
|
|
@ -351,36 +354,6 @@ in
|
|||
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
|
||||
exit 1
|
||||
'')
|
||||
(pkgs.writeShellScript "wait-nix-daemon" ''
|
||||
# Up to ~180s; the daemon is normally up within seconds, this
|
||||
# only bites in the post-restart cold window. Non-fatal shape:
|
||||
# if it never comes up the unit fails cleanly (recoverable)
|
||||
# rather than the runner claiming jobs into a dead daemon.
|
||||
#
|
||||
# `--store daemon` is load-bearing. A bare `nix store ping`
|
||||
# uses the `auto` store, which — when run as root with the
|
||||
# daemon socket absent — silently resolves to a LOCAL store
|
||||
# (root can write /nix/store directly) and pings successfully.
|
||||
# systemd service units don't source the profile that sets
|
||||
# `NIX_REMOTE=daemon`, so this is the real environment here.
|
||||
# At cold boot the daemon socket IS absent: nix-daemon.socket
|
||||
# carries `ConditionPathIsReadWrite=/nix/var/nix/daemon-socket`
|
||||
# and is condition-skipped until /nix/var goes read-write. So
|
||||
# a bare ping would pass against the local store while the
|
||||
# daemon is still down — defeating the gate's whole purpose
|
||||
# (the runner would start and claim jobs the daemon can't yet
|
||||
# service). Pinning `--store daemon` makes the poll verify the
|
||||
# actual daemon socket, so the gate honours its contract and
|
||||
# waits until the daemon — not a local fallback — answers.
|
||||
for _ in $(seq 1 90); do
|
||||
if ${pkgs.nix}/bin/nix store ping --store daemon >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "nix daemon not reachable after 180s" >&2
|
||||
exit 1
|
||||
'')
|
||||
];
|
||||
# Registration is out of band: hive-c0re explicitly restarts this
|
||||
# unit once it writes the real token, which is the primary path. As
|
||||
|
|
|
|||
Loading…
Reference in a new issue