e407fa93df
`CPUQuota=`/`MemoryMax=` are hard caps: they throttle an agent even when
the host is idle, so they are the wrong tool for "be polite under
contention". The cgroup v2 relative shares are, and neither was wired.
Adds `services.hyperhive.{agentCpuWeight,agentIoWeight}` (1..=10000,
default 80) threaded through the existing drop-in path: HiveEnv ->
write_dropins -> WriteResourceLimits -> hyperhive-limits.conf, next to
the caps already there. Hive-wide only, as the operator scoped it on the
issue: no per-agent override, no resource-limits.json field, no
dashboard form.
The default of 80 is below the kernel's 100, so agent containers yield
to everything *not* on this drop-in path -- host services and the infra
containers (hive-ci, hive-forge, hive-gateway, hive-matrix). It does not
rank agents against each other; they all carry the same weight.
`WriteResourceLimits` gains two `#[serde(default)]` fields, and the
writer treats weight 0 as "not configured" and omits the line, so an
older hive-c0re talking to a newer hive-priv still produces the exact
pre-weights drop-in. The body is extracted into `limits_dropin_body` so
that is covered by a test rather than asserted by eye.
285 lines
14 KiB
Nix
285 lines
14 KiB
Nix
# The hive-c0re coordinator daemon (runs as the unprivileged
|
|
# `hive-core` user), socket-activated at /run/hyperhive/host.sock.
|
|
# Layout: ./options.nix (option declarations), ./theme.nix (stylix
|
|
# frontend theming → `servedFrontend`), ./environment.nix (the daemon
|
|
# unit's env attrset). The root privileged helper it delegates to is
|
|
# its own module (../hive-priv.nix).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.c0re;
|
|
|
|
# Privsep splits ownership across users, so git/libgit2's dubious-
|
|
# ownership guard trips on legitimate cross-user reads: hive-priv (root)
|
|
# fetches the hive-core-owned meta/applied repos via nix, and hive-c0re
|
|
# (hive-core) fetches the agent-owned proposed-config repos. Both
|
|
# processes are trusted and can already read the files; this gitconfig
|
|
# only satisfies the ownership guard. libgit2 honours the literal `*`
|
|
# (mid-path globs aren't supported, so per-agent repos can't be listed);
|
|
# in practice these processes only ever touch hyperhive's own repos.
|
|
#
|
|
# The agent config inputs now live on the forge
|
|
# (`git+http://${forge.domain}/agent-configs/<n>.git`, see meta.rs render),
|
|
# so hive-core's `nix flake update` of those inputs is an authenticated
|
|
# `git+http` fetch. The `[credential]` stanza points git at the `hive-forge`
|
|
# helper below (scoped to the forge host) so the fetch authenticates as the
|
|
# forge `core` user with no token in any URL or lock.
|
|
safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" ''
|
|
[safe]
|
|
directory = *
|
|
[credential "http://${config.services.hyperhive.forge.domain}"]
|
|
helper = hive-forge
|
|
username = core
|
|
[http]
|
|
# Abort a stalled fetch instead of blocking startup indefinitely. If a
|
|
# git+http transfer (e.g. `nix flake lock` of the forge-hosted config
|
|
# inputs) drops below 1 KiB/s for 30s -- forge unreachable / not ready
|
|
# on cold boot -- git fails fast rather than hanging the whole daemon.
|
|
# Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the
|
|
# migration shellout timeout in migrate.rs.
|
|
lowSpeedLimit = 1024
|
|
lowSpeedTime = 30
|
|
'';
|
|
|
|
# git credential helper for hive-core's authenticated fetches of the
|
|
# agent-config repos on the forge. Reads the live forge-core admin token
|
|
# (`/var/lib/hyperhive/forge-core-token` — paths.rs `FORGE_CORE_TOKEN`) on
|
|
# every invocation, so it never holds a stale copy and survives token
|
|
# rotation. Scoped to the forge host by the `[credential]` stanza above;
|
|
# implements the git credential-helper protocol (only `get` answers).
|
|
forgeCredHelper = pkgs.writeShellScriptBin "git-credential-hive-forge" ''
|
|
[ "''${1:-}" = "get" ] || exit 0
|
|
if [ -r /var/lib/hyperhive/forge-core-token ]; then
|
|
printf 'username=core\n'
|
|
printf 'password=%s\n' "$(cat /var/lib/hyperhive/forge-core-token)"
|
|
fi
|
|
'';
|
|
|
|
# The `hive-c0re serve` config JSON. Keys are snake_case to match the
|
|
# `ServeConfig` serde shape the daemon deserialises (the
|
|
# container-injected HiveEnv fields, flattened, plus the hive-c0re-local
|
|
# model_prices table); per-flag overrides still work for ad-hoc
|
|
# invocations.
|
|
#
|
|
# Written to `/etc/hyperhive/serve.json` (managed by
|
|
# `environment.etc`) rather than embedded as a store-path argument in
|
|
# ExecStart. This keeps ExecStart byte-stable across deploys that only
|
|
# change hyperhive module files (gateway, frontend, unrelated nix
|
|
# modules) so systemd does NOT restart hive-c0re — and therefore does
|
|
# NOT trigger a startup sweep that rebuilds every agent — unless the
|
|
# c0re binary itself changes.
|
|
serveConfigJson = builtins.toJSON {
|
|
hyperhive_flake = cfg.hyperhiveFlake;
|
|
hyperhive_docs_flake = cfg.hyperhiveDocs;
|
|
nixpkgs_flake = cfg.nixpkgsFlake;
|
|
dashboard_port = cfg.dashboardPort;
|
|
operator_pronouns = cfg.operatorPronouns;
|
|
context_window_tokens = cfg.contextWindowTokens;
|
|
agent_cpu_quota = cfg.agentCpuQuota;
|
|
agent_memory_max = cfg.agentMemoryMax;
|
|
agent_cpu_weight = cfg.agentCpuWeight;
|
|
agent_io_weight = cfg.agentIoWeight;
|
|
model_prices = cfg.modelPrices;
|
|
build_slots = cfg.buildSlots;
|
|
};
|
|
in
|
|
{
|
|
imports = [
|
|
./options.nix
|
|
./theme.nix
|
|
];
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
environment.systemPackages = [
|
|
cfg.package
|
|
pkgs.git
|
|
# XDG icons + .desktop entries so desktop environments can match
|
|
# hyperhive processes to their icon (task managers, CPU monitors, etc.).
|
|
cfg.xdgIcons
|
|
];
|
|
|
|
# Serve config at a stable /etc path so hive-c0re's ExecStart
|
|
# doesn't embed a volatile store-path argument. See serveConfigJson
|
|
# above for the rationale.
|
|
environment.etc."hyperhive/serve.json".text = serveConfigJson;
|
|
|
|
# Pull the per-container toplevels into the host system closure.
|
|
# `system.extraDependencies` adds paths to the system build
|
|
# without referencing them at runtime — nixos-rebuild fetches /
|
|
# builds them, they end up in /nix/store, and the first
|
|
# nixos-container update + start for an agent has nothing left to
|
|
# do. Gated because the closure is sizeable and pinned to x86_64.
|
|
system.extraDependencies = lib.optionals cfg.preBuildAgentTemplates [
|
|
cfg.agentBaseToplevel
|
|
cfg.managerToplevel
|
|
];
|
|
|
|
# Unprivileged coordinator user. hive-c0re runs as this user;
|
|
# privileged operations are delegated to hive-priv which runs as
|
|
# root, socket-activated at /run/hive/priv.sock (./hive-priv.nix).
|
|
users.users.hive-core = {
|
|
isSystemUser = true;
|
|
group = "hive-core";
|
|
description = "hive-c0re coordinator daemon user";
|
|
};
|
|
users.groups.hive-core = { };
|
|
|
|
# Operators granted sudoless `hivectl`. Members of `hive-admin` can
|
|
# connect to the host admin socket (group-owned by hive-admin via the
|
|
# socket unit's `SocketGroup` below) without root. That socket is *full*
|
|
# hive control (spawn/kill/destroy/deploy, docs/boundary.md), so this is
|
|
# an explicit opt-in allowlist — empty by default (root-only).
|
|
users.groups.hive-admin = {
|
|
members = cfg.adminUsers;
|
|
};
|
|
|
|
# The gateway nginx is always the sole external entry point (it runs
|
|
# alongside hyperhive), so the per-agent web-port range stays closed on
|
|
# the host firewall. See `docs/gateway.md::Firewall posture (host-level)`.
|
|
|
|
# NB: `services.hyperhive.domain` is required when hyperhive is
|
|
# enabled — the canonical assertion lives in `hive-network.nix` (the
|
|
# hive resolver is authoritative for `<domain>` and agents reach the
|
|
# forge/matrix through the gateway by it). So the daemon environment
|
|
# (./environment.nix) can treat it as non-null.
|
|
systemd.services.hive-c0re = {
|
|
description = "hyperhive coordinator daemon";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Socket unit must start before the service so hive-c0re receives the
|
|
# pre-bound fd via LISTEN_FDS (socket activation). Without this
|
|
# dependency, nixos-rebuild switch activates hive-c0re.socket while
|
|
# hive-c0re.service is already running (started by multi-user.target),
|
|
# and systemd refuses with "Socket service already active". Adding
|
|
# requires+after causes systemd to stop the service, start the socket,
|
|
# then restart the service -- clean transition on every config apply.
|
|
requires = [ "hive-c0re.socket" ];
|
|
after = [ "hive-c0re.socket" ];
|
|
path = [
|
|
pkgs.git
|
|
# `git-credential-hive-forge` on PATH so git finds it when nix fetches
|
|
# the forge-hosted agent-config inputs (helper = hive-forge).
|
|
forgeCredHelper
|
|
"/run/current-system/sw"
|
|
];
|
|
environment = import ./environment.nix { inherit lib config pkgs; };
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --config /etc/hyperhive/serve.json";
|
|
SyslogIdentifier = "hive-c0re";
|
|
# Migrate hive-c0re's *own* state to the service user after an
|
|
# upgrade from a root-run install (systemd's StateDirectory only
|
|
# chowns the top-level dir, not pre-existing files inside it). The
|
|
# `+` prefix runs as root despite User = hive-core; `-` tolerates
|
|
# failure. coreutils ships `chown` but no `sh`, so invoke the
|
|
# binaries directly rather than through a shell.
|
|
#
|
|
# CRITICAL: exclude the per-agent `agents/` subtree. Its contents
|
|
# (each agent's `claude/` OAuth creds, `state/`, `harness/`,
|
|
# `config/`) are owned by the per-agent / manager users, and each
|
|
# container's `hive-agent-user-migrate` activation script chowns
|
|
# them back to that user on boot. Blanket-chowning them to hive-core
|
|
# makes every agent's `~/.claude` unreadable — logging them all out
|
|
# with no way to log back in. So chown everything *except* agents/,
|
|
# plus the `agents/` dir node itself (not its contents) so c0re can
|
|
# still create new per-agent subdirs.
|
|
ExecStartPre = [
|
|
# Install the safe.directory gitconfig at $HOME/.gitconfig
|
|
# (HOME = /var/lib/hyperhive) so c0re's `git fetch`/`rev-parse`
|
|
# against the agent-owned proposed repos pass the ownership guard.
|
|
# Placed before the chown below so it's chowned to hive-core too.
|
|
"+-${pkgs.coreutils}/bin/cp ${safeDirGitconfig} /var/lib/hyperhive/.gitconfig"
|
|
"+-${pkgs.findutils}/bin/find /var/lib/hyperhive -mindepth 1 -maxdepth 1 -not -name agents -exec ${pkgs.coreutils}/bin/chown -R hive-core:hive-core {} +"
|
|
"+-${pkgs.coreutils}/bin/chown hive-core:hive-core /var/lib/hyperhive/agents"
|
|
];
|
|
Restart = "on-failure";
|
|
RestartSec = 2;
|
|
User = "hive-core";
|
|
Group = "hive-core";
|
|
SupplementaryGroups = [ "systemd-journal" ];
|
|
RuntimeDirectory = "hyperhive";
|
|
# 0751 (traverse-only, no listing) so `hive-admin` operators can reach
|
|
# the host admin socket (`SocketGroup = "hive-admin"`, 0660) without
|
|
# root. Others can traverse but not list; the socket + per-agent
|
|
# subdirs gate access by their own perms. Matches the socket unit's
|
|
# DirectoryMode.
|
|
RuntimeDirectoryMode = "0751";
|
|
RuntimeDirectoryPreserve = "yes";
|
|
StateDirectory = "hyperhive";
|
|
StateDirectoryMode = "0750";
|
|
# OTEL auth-header secret, loaded onto hive-c0re's own unit so its
|
|
# per-agent container-resource metrics exporter can read it
|
|
# at $CREDENTIALS_DIRECTORY/otel-headers — via systemd, not a world
|
|
# path. Same secret the agent containers get (forwarded there via
|
|
# nspawn --load-credential); this just also hands it to c0re itself.
|
|
# Empty list (no credential) when otel is off or no header is set.
|
|
LoadCredential = lib.optional (
|
|
config.services.hyperhive.otel.enable && config.services.hyperhive.otel.headersCredential != null
|
|
) "otel-headers:${config.services.hyperhive.otel.headersCredential}";
|
|
# Sandboxing. hive-c0re is unprivileged (runs as hive-core, never
|
|
# setuid), makes HTTP requests to forge/matrix/Anthropic (keeps INET),
|
|
# and delegates all privileged ops to hive-priv via a Unix socket.
|
|
# These directives deny the subset of kernel capabilities it
|
|
# provably doesn't need without restricting its network or
|
|
# filesystem access (RestrictAddressFamilies deferred — needs a
|
|
# watched deploy to verify no AF_UNIX/AF_INET gaps in socket paths).
|
|
NoNewPrivileges = true; # already runs as unprivileged user
|
|
PrivateTmp = true; # uses StateDirectory for tmpfiles, not /tmp
|
|
ProtectHome = true; # HOME = /var/lib/hyperhive; no /home/* access needed
|
|
# "strict" makes the entire filesystem read-only except for
|
|
# StateDirectory (/var/lib/hyperhive) and RuntimeDirectory
|
|
# (/run/hyperhive), which systemd keeps writable. No
|
|
# ReadWritePaths needed beyond the managed directories because:
|
|
# - nix is invoked directly (lifecycle, meta, flake_check), but
|
|
# NIX_REMOTE=daemon routes all store writes through the host
|
|
# daemon — hive-c0re never writes to /nix itself.
|
|
# - flake.lock ops land in the meta worktree under StateDirectory
|
|
# (kept writable by systemd).
|
|
# - nix build worktrees live in PrivateTmp, not /tmp.
|
|
# - /etc writes (bind-mount edits) go through hive-priv via the
|
|
# privileged socket; /etc/hyperhive/serve.json is read-only.
|
|
ProtectSystem = "strict";
|
|
ProtectKernelTunables = true; # no sysctl writes
|
|
ProtectKernelLogs = true; # reads logs via systemd-journal group, not /dev/kmsg
|
|
ProtectControlGroups = true; # cgroup writes go through hive-priv, not c0re directly
|
|
RestrictNamespaces = true; # namespace creation goes through hive-priv
|
|
LockPersonality = true; # no personality changes needed
|
|
RestrictRealtime = true; # no real-time scheduling
|
|
};
|
|
};
|
|
|
|
# Socket unit for the hive-c0re admin socket. systemd creates and holds
|
|
# `/run/hyperhive/host.sock` before hive-c0re starts, then passes the fd
|
|
# via LISTEN_FDS (socket activation). Benefits: `hivectl` can connect
|
|
# the moment the socket unit is active — no racy retry window — and a
|
|
# hive-c0re restart never drops the socket inode, so queued commands
|
|
# drain cleanly.
|
|
#
|
|
# `hive-c0re serve` reads LISTEN_FDS via the `listenfd` crate and
|
|
# accepts the fd in preference to its own `bind()` path. When invoked
|
|
# directly (dev, CI, without the socket unit) LISTEN_FDS is absent and
|
|
# the traditional bind path runs unchanged — no regression.
|
|
systemd.sockets.hive-c0re = {
|
|
description = "hive-c0re admin socket";
|
|
wantedBy = [ "sockets.target" ];
|
|
socketConfig = {
|
|
# Must match the `--socket` arg passed to `hive-c0re serve`.
|
|
ListenStream = "/run/hyperhive/host.sock";
|
|
# `0660 root:hive-admin` — group-owned by `hive-admin` so operators in
|
|
# that group (services.hyperhive.c0re.adminUsers) drive `hivectl`
|
|
# without root; an empty adminUsers list leaves the group memberless,
|
|
# so it stays effectively root-only. See docs/boundary.md.
|
|
SocketMode = "0660";
|
|
SocketGroup = "hive-admin";
|
|
# `0751` (traverse-only, no listing) so hive-admin can reach the socket
|
|
# path — the socket's own `0660 hive-admin` gates the connection, and
|
|
# the per-agent subdirs under here keep their own restrictive perms.
|
|
# Must match the service unit's RuntimeDirectoryMode. DirectoryMode is
|
|
# only consulted when the dir is absent at socket-unit activation.
|
|
DirectoryMode = "0751";
|
|
};
|
|
};
|
|
};
|
|
}
|