65a0686297
Container nix invocations hard-failed whenever the remote builder muede-pc2 was unreachable, while the identical build on the host succeeded. Both go through the same host nix-daemon, so the difference looked impossible. The cause is that `fallback` is a client-side option: the nix client transmits it to the daemon per connection (`tryFallback` in setOptions), so the caller decides whether a failed remote dispatch may degrade to a local build — even when the build itself runs on the host daemon under NIX_REMOTE=daemon. Only genuinely daemon-side settings such as `builders` are inherited from the host. The containers never set `fallback`, so they took nix's default of false. Set it in the agent-container base module and in the CI container, and correct the hive-ci comment that claimed fallback was inherited from the host daemon along with buildMachines and max-jobs. Verified in an agent container: `nix fmt` fails outright on the remote store's connection reset, while the same command with fallback enabled reports the same connection error and then builds locally and succeeds.
237 lines
10 KiB
Nix
237 lines
10 KiB
Nix
# Shared scaffolding for every hyperhive harness container.
|
|
# `../templates/agent.nix` and `../templates/ruth.nix` both import
|
|
# agents use the same service unit regardless of which entry-point
|
|
# they came from.
|
|
#
|
|
# This is the core module: container plumbing (boot/nix/nixpkgs),
|
|
# base tooling, and the cross-cutting `hyperhive.icon` option. Each
|
|
# feature lives in its own sibling module (imported below) that
|
|
# declares its own `hyperhive.*` options + config.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
# Flake inputs routed through _module.args by the agent flake.nix.
|
|
# Default to {} so the module evaluates cleanly even when the agent
|
|
# flake doesn't set up the routing pattern (e.g. during standalone
|
|
# nixos-rebuild without a flake wrapper).
|
|
flakeInputs ? { },
|
|
...
|
|
}:
|
|
{
|
|
imports = [
|
|
./agent-service.nix
|
|
./bash-env.nix
|
|
./claude-settings.nix
|
|
./dashboard-links.nix
|
|
./docs.nix
|
|
./forge.nix
|
|
./frontend.nix
|
|
./github.nix
|
|
./matrix.nix
|
|
./mcp.nix
|
|
./network.nix
|
|
./packages.nix
|
|
./user.nix
|
|
./screen.nix
|
|
./weston-vnc.nix
|
|
(lib.mkRemovedOptionModule [ "hyperhive" "web" "useUnixSocket" ] ''
|
|
Unix socket mode is always enabled for all agents. Remove the
|
|
setting from your agent.nix.
|
|
'')
|
|
(lib.mkRemovedOptionModule [ "hyperhive" "allowedBashPatterns" ] ''
|
|
The built-in Bash tool is fully disabled; agents use
|
|
mcp__bash__run instead. Remove the setting from your agent.nix.
|
|
'')
|
|
];
|
|
|
|
options.hyperhive.icon = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.path;
|
|
default = null;
|
|
example = lib.literalExpression "./icon.svg";
|
|
description = ''
|
|
Path to an SVG file used as this agent's icon — shown on the
|
|
dashboard and the per-agent web UI (header + favicon). Commit
|
|
the SVG into the agent's config repo next to `agent.nix` and
|
|
reference it as a relative path (`./icon.svg`).
|
|
|
|
When null (the default) the agent falls back to the shared
|
|
hyperhive logo. The harness serves the icon (configured or
|
|
default) at `GET /icon` on the per-agent web port.
|
|
'';
|
|
};
|
|
|
|
config = {
|
|
assertions = [
|
|
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
|
|
# expected to set `_module.args.flakeInputs = builtins.removeAttrs inputs ["self"]`.
|
|
# If `self` leaks into flakeInputs the agent gets a spurious attrset
|
|
# entry that can shadow real inputs and is almost certainly a bug.
|
|
# Guard with `or {}` so standalone evaluation stays clean when
|
|
# flakeInputs is absent from _module.args.
|
|
{
|
|
assertion = !(builtins.hasAttr "self" (config._module.args.flakeInputs or { }));
|
|
message = ''
|
|
hyperhive: `flakeInputs` must not contain "self".
|
|
In your agent flake.nix, use:
|
|
_module.args.flakeInputs = builtins.removeAttrs inputs [ "self" ];
|
|
'';
|
|
}
|
|
# hyperhive.icon must reference an SVG file when set.
|
|
{
|
|
assertion = config.hyperhive.icon == null || lib.hasSuffix ".svg" (toString config.hyperhive.icon);
|
|
message = "hyperhive.icon must point to an .svg file";
|
|
}
|
|
];
|
|
|
|
# Operator-set per-agent icon (hyperhive.icon). When configured, the
|
|
# SVG lands at /etc/hyperhive/icon.svg; the harness serves it at
|
|
# GET /icon, falling back to the bundled hyperhive logo when absent.
|
|
# Consumed by forge-avatar-sync (./forge.nix) and the matrix avatar
|
|
# sync (./matrix.nix) too.
|
|
environment.etc."hyperhive/icon.svg" = lib.mkIf (config.hyperhive.icon != null) {
|
|
source = config.hyperhive.icon;
|
|
};
|
|
|
|
boot.isNspawnContainer = true;
|
|
|
|
# Use a disk-backed /tmp instead of the default tmpfs so large scratch
|
|
# writes (nix-develop shells, cargo build dirs, multi-GB downloads) land
|
|
# on disk rather than eating container RAM. The tmpfs default mounts
|
|
# ~3.2 GB of RAM per container; disk-backed /tmp is effectively unlimited
|
|
# and cheaper for agents that do heavy build work.
|
|
#
|
|
# cleanOnBoot defaults to false in nixpkgs — set it explicitly so /tmp is
|
|
# cleared on each container start (D! tmpfiles rule), preserving the
|
|
# ephemeral-per-boot semantics agents expect from a tmpfs /tmp, just
|
|
# without the RAM cost.
|
|
boot.tmp.useTmpfs = false;
|
|
boot.tmp.cleanOnBoot = true;
|
|
|
|
# Every agent gets flakes + the modern `nix` CLI out of the box.
|
|
# Equivalent to passing `--extra-experimental-features 'nix-command
|
|
# flakes'` on every invocation. Agents shell out to `nix build` /
|
|
# `nix flake` constantly (devshells, ad-hoc evals, fetching their
|
|
# own MCP-server flakes); without this they hit the "experimental
|
|
# feature not enabled" wall on the first try.
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# `lib.mkForce` overrides nixpkgs's normal-priority `false` so
|
|
# in-container `nix build` invocations fall back to unsandboxed
|
|
# local builds rather than failing on the missing user-namespace.
|
|
# See `docs/gotchas.md::Containerized nix-daemon needs
|
|
# sandbox-fallback = true` + `docs/security.md` for the rationale.
|
|
#
|
|
# Note: with NIX_REMOTE=daemon below this becomes a no-op for the
|
|
# common case — daemon-routed builds run on the host where sandboxing
|
|
# works. It stays as a belt-and-suspenders fallback for any context
|
|
# that bypasses the daemon (e.g. direct nix-store invocations).
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
|
|
# Fall back to a local build when a remote builder can't be reached,
|
|
# instead of hard-failing the whole invocation.
|
|
#
|
|
# This is NOT redundant with the host daemon's own setting, and that
|
|
# is the subtle part: `fallback` (protocol `tryFallback`) is a *client*
|
|
# option. Every nix client transmits it to the daemon on connect, so
|
|
# the client — i.e. this container — decides whether a failed remote
|
|
# dispatch may degrade to a local build, even though the build itself
|
|
# executes on the host daemon under NIX_REMOTE=daemon below. Contrast
|
|
# `builders`, which is genuinely daemon-side and which an untrusted
|
|
# client cannot override. Without this, a container inherits nix's
|
|
# default `false`, so a momentarily unreachable remote builder kills
|
|
# the invocation while the exact same build started on the host
|
|
# succeeds.
|
|
nix.settings.fallback = true;
|
|
|
|
# Route ALL nix invocations in this container through the host
|
|
# nix-daemon socket, regardless of whether the caller is root or
|
|
# non-root. Without this, root contexts (PID 1, systemd services
|
|
# running as root) default to store=auto which resolves to the LOCAL
|
|
# store — bypassing the shared daemon, its remote builders, and the
|
|
# host's prebuilt derivation cache, causing spurious full rebuilds.
|
|
#
|
|
# systemd.globalEnvironment sets DefaultEnvironment in systemd.conf,
|
|
# so every unit started by PID 1 inherits NIX_REMOTE=daemon.
|
|
# Non-root nix clients already default to the daemon socket, so this
|
|
# is a no-op for them; it only matters for root services that would
|
|
# otherwise silently use the local store.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
|
|
# `claude-code` is unfree. Each per-agent container's nixosConfiguration
|
|
# evaluates its own `nixpkgs` instance, so the operator's host-level
|
|
# `nixpkgs.config.allowUnfreePredicate` does not propagate into here —
|
|
# we have to allow it inside the container's config as well.
|
|
nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "claude-code" ];
|
|
|
|
# Core tooling every agent gets. Per-bin split packages (see
|
|
# nix/packages/default.nix + ./packages.nix) rather than the full
|
|
# `hyperhive` bundle — that bundle also carries `hivectl` (a
|
|
# host-admin CLI that dials the *host* admin socket — useless and
|
|
# unreachable from inside a container — wrapped with
|
|
# `wireguard-tools` for `hivectl wg`). The daemon/harness/MCP bins
|
|
# the harness execs (hive-agent{,-mcp}, hive-bash-daemon,
|
|
# hive-matrix-daemon, hive-matrix-mcp) are wired via their own
|
|
# ExecStart/command lines in the sibling modules — they don't need
|
|
# to be on PATH too. Only these two are actually looked up on PATH
|
|
# by claude/shell code inside the container:
|
|
# `hive-agent-wake` (external wake CLI, docs/turn-loop/mcp.md) and
|
|
# `hive-metric` (agent-emitted custom metrics CLI,
|
|
# docs/observability.md).
|
|
environment.systemPackages = [
|
|
config.hyperhive.packages.hive-agent-wake
|
|
config.hyperhive.packages.hive-metric
|
|
]
|
|
++ (with pkgs; [
|
|
claude-code
|
|
bashInteractive
|
|
coreutils-full
|
|
# procps for pkill — used by the web UI's /api/cancel to SIGINT the
|
|
# in-flight claude turn.
|
|
procps
|
|
# jq: JSON processing in shell — useful for parsing API responses,
|
|
# forge REST calls, sqlite output, etc.
|
|
jq
|
|
# curl: HTTP client for forge REST API and other web requests.
|
|
curl
|
|
]);
|
|
|
|
# HIVE_ASSETS_DIR points at the project's static runtime assets
|
|
# (branding + claude prompts; see `nix/packages/assets.nix`). Set
|
|
# here so both the harness binary and any user-shell `cargo run`
|
|
# inside the container resolve them from the same path.
|
|
# SHELL must be set so claude's Bash tool finds a POSIX shell.
|
|
# HIVE_CONTEXT_WINDOW_TOKENS_* are injected by the meta flake from the
|
|
# host-level `services.hyperhive.c0re.contextWindowTokens` option — not
|
|
# set here.
|
|
environment.variables = {
|
|
HIVE_ASSETS_DIR = "${config.hyperhive.packages.assets}/share/hyperhive";
|
|
SHELL = "${pkgs.bashInteractive}/bin/bash";
|
|
# Route interactive-shell nix invocations through the host daemon.
|
|
# Redundant with /etc/profile.d/nix-daemon.sh but ensures it's set
|
|
# regardless of which profile files are sourced.
|
|
NIX_REMOTE = "daemon";
|
|
};
|
|
|
|
# Git is needed by claude's Bash tool (for the agent <-> manager config
|
|
# request flow) and by hive-c0re's own setup_applied / setup_proposed.
|
|
# The per-agent `applied/<name>/flake.nix` overrides `user.name` and
|
|
# `user.email` with the agent's identity — values here are `mkDefault`
|
|
# so the per-agent override wins without needing `mkForce`.
|
|
programs.git = {
|
|
enable = true;
|
|
config = {
|
|
user = {
|
|
name = lib.mkDefault "hyperhive";
|
|
email = lib.mkDefault "hyperhive@local";
|
|
};
|
|
init.defaultBranch = lib.mkDefault "main";
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "25.11";
|
|
};
|
|
}
|