fix(#1748): route all nix invocations through the host daemon (NIX_REMOTE=daemon)
Root contexts (systemd services running as root, PID 1) default to store=auto which resolves to the LOCAL nix store — bypassing the host daemon, its remote builders (muede-pc2), and any prebuilt derivation outputs already in the shared store. This causes spurious full rebuilds of agent toplevels that the host already built and cached. Two changes: harness-base.nix: - Add systemd.globalEnvironment.NIX_REMOTE = "daemon" — sets DefaultEnvironment in systemd.conf so every unit in the container inherits NIX_REMOTE=daemon. Non-root contexts already default to the daemon socket; this only matters for root services that would otherwise use the local store. - Add NIX_REMOTE = "daemon" to environment.variables so interactive shells also have it set (redundant with /etc/profile.d/nix-daemon.sh but explicit and profile-agnostic). hive-c0re.nix (hive-priv service): - Add NIX_REMOTE = "daemon" to the service environment. hive-priv runs as root and invokes nixos-container update + nix prebuild; these must route through the host daemon so they see the shared store and remote builders, not a private local store. The sandbox-fallback = true in harness-base.nix is kept as a belt- and-suspenders fallback but becomes a no-op for the common case once nix routes through the daemon (the daemon builds on the host where sandboxing works).
This commit is contained in:
parent
3e5ddbd2c0
commit
3a4b8d9873
2 changed files with 29 additions and 0 deletions
|
|
@ -1215,6 +1215,12 @@ in
|
|||
# errors out. Point HOME at the StateDirectory below (persistent,
|
||||
# so the cache survives across rebuilds).
|
||||
HOME = "/var/lib/hive-priv";
|
||||
# hive-priv runs as root. Root nix defaults to store=auto which
|
||||
# resolves to the LOCAL store — bypassing the host daemon, its
|
||||
# remote builders, and prebuilt derivation outputs. Force daemon
|
||||
# routing so nixos-container update and the nix prebuild see the
|
||||
# same store and substituters as every other build context.
|
||||
NIX_REMOTE = "daemon";
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/hive-priv";
|
||||
|
|
|
|||
|
|
@ -1394,6 +1394,10 @@ in
|
|||
HIVE_DEFAULT_EFFORT = config.hyperhive.effortLevel;
|
||||
HIVE_ASSETS_DIR = "${pkgs.hyperhive-assets}/share/hyperhive";
|
||||
SHELL = "${pkgs.bashInteractive}/bin/bash";
|
||||
# Route interactive-shell nix invocations through the host daemon.
|
||||
# Redundant with /etc/profile.d/nix-daemon.sh but ensures it's set
|
||||
# regardless of which profile files are sourced.
|
||||
NIX_REMOTE = "daemon";
|
||||
}
|
||||
// lib.optionalAttrs (!config.hyperhive.autoCompact) {
|
||||
# Zero watermark disables proactive compaction; the reactive path
|
||||
|
|
@ -1449,8 +1453,27 @@ in
|
|||
# local builds rather than failing on the missing user-namespace.
|
||||
# See `docs/gotchas.md::Containerized nix-daemon needs
|
||||
# sandbox-fallback = true` + `docs/security.md` for the rationale.
|
||||
#
|
||||
# Note: with NIX_REMOTE=daemon below this becomes a no-op for the
|
||||
# common case — daemon-routed builds run on the host where sandboxing
|
||||
# works. It stays as a belt-and-suspenders fallback for any context
|
||||
# that bypasses the daemon (e.g. direct nix-store invocations).
|
||||
nix.settings.sandbox-fallback = lib.mkForce true;
|
||||
|
||||
# Route ALL nix invocations in this container through the host
|
||||
# nix-daemon socket, regardless of whether the caller is root or
|
||||
# non-root. Without this, root contexts (PID 1, systemd services
|
||||
# running as root) default to store=auto which resolves to the LOCAL
|
||||
# store — bypassing the shared daemon, its remote builders, and the
|
||||
# host's prebuilt derivation cache, causing spurious full rebuilds.
|
||||
#
|
||||
# systemd.globalEnvironment sets DefaultEnvironment in systemd.conf,
|
||||
# so every unit started by PID 1 inherits NIX_REMOTE=daemon.
|
||||
# Non-root nix clients already default to the daemon socket, so this
|
||||
# is a no-op for them; it only matters for root services that would
|
||||
# otherwise silently use the local store.
|
||||
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
||||
|
||||
# `claude-code` is unfree. Each per-agent container's nixosConfiguration
|
||||
# evaluates its own `nixpkgs` instance, so the operator's host-level
|
||||
# `nixpkgs.config.allowUnfreePredicate` does not propagate into here —
|
||||
|
|
|
|||
Loading…
Reference in a new issue