41a7974b9e
The external export is a setting of the swarm collector, so it moves to services.hyperhive.deploy.swarm-otel.upstream. Renamed: - otel.endpoint -> upstream.endpoint - otel.protocol -> upstream.protocol - otel.headersCredential -> upstream.headersCredential - otel.collector.upstreamHeaderName -> upstream.headerName The unreleased otel.upstreamSignals is declared as upstream.signals. Hosts keep working through the rename warnings. Eval test covers the old paths reaching the new options. Docs and stale comments point at the new paths.
414 lines
21 KiB
Nix
414 lines
21 KiB
Nix
# The hive-c0re coordinator daemon (runs as the unprivileged
|
|
# `hive-core` user), socket-activated at /run/hyperhive/host.sock.
|
|
# Layout: ./options.nix (option declarations), ./theme.nix (stylix
|
|
# frontend theming → `servedFrontend`), ./vhosts.nix (its gateway vhosts),
|
|
# ./environment.nix (the daemon unit's env attrset). The root privileged
|
|
# helper it delegates to is its own module (../hive-priv.nix).
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.c0re;
|
|
controllerCfg = config.services.hyperhive.deploy.hive-controller;
|
|
|
|
# `/run/hyperhive`'s mode. Held in one place because whichever of the two
|
|
# units below activates first creates the directory, so they cannot be
|
|
# allowed to disagree about what it should be. `0751` is traverse-only, no
|
|
# listing: `hive-admin` members reach `host.sock` without root, while the
|
|
# socket's own `0660 hive-admin` gates the connection and the per-agent
|
|
# subdirs keep their own perms.
|
|
runtimeDirMode = "0751";
|
|
|
|
baoDeploy = config.services.hyperhive.deploy.bao;
|
|
# Held in one place because the LoadCredential below and the `BAO_*`
|
|
# environment in ./environment.nix have to agree on when they exist: a
|
|
# credential with no reader is dead weight, and an environment naming a
|
|
# credential nobody loaded is a daemon that fails at the TLS handshake.
|
|
baoClientIdentity = baoDeploy.clientCertFile != null && baoDeploy.clientKeyFile != null;
|
|
|
|
caTrust = import ../lib/hive-ca-trust.nix {
|
|
inherit lib;
|
|
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
};
|
|
|
|
# Privsep splits ownership across users, so git/libgit2's dubious-
|
|
# ownership guard trips on legitimate cross-user reads: hive-priv (root)
|
|
# fetches the hive-core-owned meta/applied repos via nix, and hive-c0re
|
|
# (hive-core) fetches the agent-owned proposed-config repos. Both
|
|
# processes are trusted and can already read the files; this gitconfig
|
|
# only satisfies the ownership guard. libgit2 honours the literal `*`
|
|
# (mid-path globs aren't supported, so per-agent repos can't be listed);
|
|
# in practice these processes only ever touch hyperhive's own repos.
|
|
#
|
|
# The agent config inputs are `git+http` forge URLs pinned by sha (meta.rs
|
|
# render). The `[credential]` stanza points git at the `hive-forge` helper
|
|
# below, scoped to the forge host, for any `git+http` fetch from it as the
|
|
# forge `core` user. hive-c0re's own forge reads use `forge::reader_token()` instead.
|
|
#
|
|
# git runs every matching `credential.helper`, system config first, so a
|
|
# host helper (e.g. `oauth`, which waits on a browser) would run ahead of
|
|
# ours. An empty `helper =` resets the list; it comes first at both the
|
|
# global and the forge level so no /etc/gitconfig helper survives.
|
|
safeDirGitconfig = pkgs.writeText "hyperhive-safe-gitconfig" ''
|
|
[safe]
|
|
directory = *
|
|
[credential]
|
|
helper =
|
|
[credential "http://${config.services.hyperhive.swarm.forge.domain}"]
|
|
helper =
|
|
helper = hive-forge
|
|
username = core
|
|
[http]
|
|
# Abort a stalled fetch instead of blocking startup indefinitely. If a
|
|
# git http transfer from the forge (e.g. filling an applied mirror)
|
|
# drops below 1 KiB/s for 30s -- forge unreachable / not ready on cold
|
|
# boot -- git fails fast rather than hanging the whole daemon.
|
|
# Pairs with GIT_TERMINAL_PROMPT=0 (no credential-prompt hang) and the
|
|
# migration shellout timeout in migrate.rs.
|
|
lowSpeedLimit = 1024
|
|
lowSpeedTime = 30
|
|
'';
|
|
|
|
# git credential helper answering for the forge host as `core`. Reads the
|
|
# live forge-core admin token
|
|
# (`/var/lib/hyperhive/forge-core-token` — paths.rs `FORGE_CORE_TOKEN`) on
|
|
# every invocation, so it never holds a stale copy and survives token
|
|
# rotation; prints nothing where that file is absent. Scoped to the forge
|
|
# host by the `[credential]` stanza above; implements the git
|
|
# credential-helper protocol (only `get` answers).
|
|
forgeCredHelper = pkgs.writeShellScriptBin "git-credential-hive-forge" ''
|
|
[ "''${1:-}" = "get" ] || exit 0
|
|
if [ -r /var/lib/hyperhive/forge-core-token ]; then
|
|
printf 'username=core\n'
|
|
printf 'password=%s\n' "$(cat /var/lib/hyperhive/forge-core-token)"
|
|
fi
|
|
'';
|
|
|
|
# The `hive-c0re serve` config JSON. Keys are snake_case to match the
|
|
# `ServeConfig` serde shape the daemon deserialises (the
|
|
# container-injected HiveEnv fields, flattened, plus the hive-c0re-local
|
|
# model_prices table); per-flag overrides still work for ad-hoc
|
|
# invocations.
|
|
#
|
|
# Written to `/etc/hyperhive/serve.json` (managed by
|
|
# `environment.etc`) rather than embedded as a store-path argument in
|
|
# ExecStart. This keeps ExecStart byte-stable across deploys that only
|
|
# change hyperhive module files (gateway, frontend, unrelated nix
|
|
# modules) so systemd does NOT restart hive-c0re — and therefore does
|
|
# NOT trigger a startup sweep that rebuilds every agent — unless the
|
|
# c0re binary itself changes.
|
|
serveConfigJson = builtins.toJSON {
|
|
hyperhive_flake = cfg.hyperhiveFlake;
|
|
hyperhive_rev = cfg.hyperhiveRev;
|
|
# The `claude-code` every agent runs, or null for "each agent keeps
|
|
# the one out of its own nixpkgs". `builtins.toJSON` serialises a
|
|
# derivation as its out path (and null as null), so the package goes
|
|
# in whole rather than interpolated — that is also the ONLY thing
|
|
# gc-rooting it: the resulting string carries store context, so this
|
|
# /etc entry genuinely references the package and the host's system
|
|
# closure holds it alive. Nothing container-side can — meta.rs writes
|
|
# the path into each agent's generated flake as a plain string
|
|
# literal, and text is not a reference. Hence the assertion below: do
|
|
# NOT discard this context, and do not route the path through
|
|
# anything that drops it. The failure mode is a garbage-collected
|
|
# `claude` and a hive that can't take a turn, weeks after the commit
|
|
# that caused it.
|
|
claude_code_path =
|
|
if controllerCfg.claudeCodePackage == null then
|
|
null
|
|
else
|
|
import ../../claude-code-no-alsa.nix lib controllerCfg.claudeCodePackage;
|
|
dashboard_port = controllerCfg.dashboardPort;
|
|
agent_cpu_quota = cfg.agentCpuQuota;
|
|
agent_memory_max = cfg.agentMemoryMax;
|
|
agent_cpu_weight = cfg.agentCpuWeight;
|
|
agent_io_weight = cfg.agentIoWeight;
|
|
model_prices = cfg.modelPrices;
|
|
build_slots = cfg.buildSlots;
|
|
};
|
|
in
|
|
{
|
|
imports = [
|
|
./options.nix
|
|
./theme.nix
|
|
./vhosts.nix
|
|
# Hive-CA trust for this daemon's outbound TLS. The matrix provisioning
|
|
# URL (`swarm.matrix.apiUrl`) defaults to https on the homeserver's
|
|
# gateway vhost, which serves a leaf chaining to the store's pki root —
|
|
# a CA generated at runtime, which no build-time trust store can
|
|
# contain. The queue's TLS is verified against `HIVE_C0RE_OIDC_CA_FILE`
|
|
# (../glue-hive-tls-controllers.nix), not this bundle.
|
|
#
|
|
# ⚠️ `SSL_CERT_FILE` REPLACES the trust store, and unlike the other
|
|
# consumers of this helper, hive-c0re already makes a *public*-TLS call:
|
|
# the OTEL exporter reaches whatever
|
|
# `deploy.swarm-otel.upstream.endpoint` names, typically a
|
|
# normal internet host. The bundle is system CAs + the store's root
|
|
# precisely so that path keeps working — narrowing it to the store's root
|
|
# alone would fix a case nobody hits yet and break one that runs today.
|
|
(caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = "hive-c0re";
|
|
consumers = [ "hive-c0re" ];
|
|
hostUnit = true;
|
|
enable = config.services.hyperhive.deploy.hive-controller.enable;
|
|
})
|
|
];
|
|
|
|
config = lib.mkIf config.services.hyperhive.deploy.hive-controller.enable {
|
|
# This module is the host's only knowledge that agent containers
|
|
# exist: they hang off the bridge, resolve through dnsmasq, and their
|
|
# UIs — plus the operator dashboard — are served by the gateway.
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
|
|
assertions = [
|
|
{
|
|
# The pinned claude reaches agents as a bare path, so this
|
|
# string's store context is the whole reason the binary survives
|
|
# a `nix-collect-garbage`. Losing it is invisible at eval and at
|
|
# deploy — it only shows up as every agent failing to spawn
|
|
# `claude`, at whatever unrelated moment the gc runs. Cheap
|
|
# enough to just check.
|
|
assertion =
|
|
controllerCfg.claudeCodePackage == null
|
|
|| builtins.hasContext (builtins.toJSON controllerCfg.claudeCodePackage);
|
|
message = ''
|
|
services.hyperhive.deploy.hive-controller.claudeCodePackage lost its store
|
|
context on the way into /etc/hyperhive/serve.json, so the
|
|
package is no longer gc-rooted by the system closure and
|
|
`nix-collect-garbage` may delete the claude every agent runs.
|
|
Something on that path discarded the context (e.g.
|
|
builtins.unsafeDiscardStringContext, toString, or reading the
|
|
path back out of a plain file) — undo it.
|
|
'';
|
|
}
|
|
];
|
|
|
|
environment.systemPackages = [
|
|
cfg.package
|
|
pkgs.git
|
|
# XDG icons + .desktop entries so desktop environments can match
|
|
# hyperhive processes to their icon (task managers, CPU monitors, etc.).
|
|
cfg.xdgIcons
|
|
];
|
|
|
|
# Serve config at a stable /etc path so hive-c0re's ExecStart
|
|
# doesn't embed a volatile store-path argument. See serveConfigJson
|
|
# above for the rationale.
|
|
environment.etc."hyperhive/serve.json".text = serveConfigJson;
|
|
|
|
# Pull the agent-base toplevel into the host system closure.
|
|
# `system.extraDependencies` adds paths to the system build
|
|
# without referencing them at runtime — nixos-rebuild fetches /
|
|
# builds them, they end up in /nix/store, and the first
|
|
# nixos-container update + start for an agent has nothing left to
|
|
# do. Gated because the closure is sizeable and pinned to x86_64.
|
|
system.extraDependencies = lib.optionals cfg.preBuildAgentTemplates [
|
|
cfg.agentBaseToplevel
|
|
];
|
|
|
|
# Unprivileged coordinator user. hive-c0re runs as this user;
|
|
# privileged operations are delegated to hive-priv which runs as
|
|
# root, socket-activated at /run/hive/priv.sock (./hive-priv.nix).
|
|
users.users.hive-core = {
|
|
isSystemUser = true;
|
|
group = "hive-core";
|
|
description = "hive-c0re coordinator daemon user";
|
|
};
|
|
users.groups.hive-core = { };
|
|
|
|
# Operators granted sudoless `hivectl`. Members of `hive-admin` can
|
|
# connect to the host admin socket (group-owned by hive-admin via the
|
|
# socket unit's `SocketGroup` below) without root. That socket is *full*
|
|
# hive control (spawn/kill/destroy/deploy, docs/trust-boundary/boundary.md), so this is
|
|
# an explicit opt-in allowlist — empty by default (root-only).
|
|
users.groups.hive-admin = {
|
|
members = cfg.adminUsers;
|
|
};
|
|
|
|
# Let `hive-admin` members run `hivectl agent <name> choom`
|
|
# (`machinectl shell`) without sudo, same opt-in-by-group shape as the
|
|
# socket grant above. `machinectl shell <name>@h-<name>` triggers exactly
|
|
# one polkit action, `org.freedesktop.machine1.shell` ("Acquire a shell
|
|
# in a local container") — not `.login` (a different verb, `machinectl
|
|
# login`, which `choom` never calls) and not `.host-shell` (the
|
|
# host-target variant, for a bare `machinectl shell` with no `@machine`
|
|
# suffix; `choom` always targets `<name>@h-<name>`). `enable = true` is
|
|
# required here: `security.polkit.extraConfig` is silently dropped
|
|
# unless the module itself is turned on, and nothing else in a headless
|
|
# hive pulls polkit in the way a desktop session would.
|
|
security.polkit.enable = true;
|
|
security.polkit.extraConfig = ''
|
|
polkit.addRule(function(action, subject) {
|
|
if (action.id == "org.freedesktop.machine1.shell" &&
|
|
subject.isInGroup("hive-admin")) {
|
|
return polkit.Result.YES;
|
|
}
|
|
});
|
|
'';
|
|
|
|
# The gateway nginx is always the sole external entry point (it runs
|
|
# alongside hyperhive), so the per-agent web-port range stays closed on
|
|
# the host firewall. See `docs/networking/gateway.md::Firewall posture (host-level)`.
|
|
|
|
# NB: `services.hyperhive.domain` is required when hyperhive is
|
|
# enabled — the canonical assertion lives in `hive-network.nix` (the
|
|
# hive resolver is authoritative for `<domain>` and agents reach the
|
|
# forge/matrix through the gateway by it). So the daemon environment
|
|
# (./environment.nix) can treat it as non-null.
|
|
systemd.services.hive-c0re = {
|
|
description = "hyperhive coordinator daemon";
|
|
wantedBy = [ "multi-user.target" ];
|
|
# Socket unit must start before the service so hive-c0re receives the
|
|
# pre-bound fd via LISTEN_FDS (socket activation). Without this
|
|
# dependency, nixos-rebuild switch activates hive-c0re.socket while
|
|
# hive-c0re.service is already running (started by multi-user.target),
|
|
# and systemd refuses with "Socket service already active". Adding
|
|
# requires+after causes systemd to stop the service, start the socket,
|
|
# then restart the service -- clean transition on every config apply.
|
|
requires = [ "hive-c0re.socket" ];
|
|
after = [ "hive-c0re.socket" ];
|
|
path = [
|
|
pkgs.git
|
|
# `git-credential-hive-forge` on PATH so git finds it for git+http
|
|
# fetches from the forge (helper = hive-forge).
|
|
forgeCredHelper
|
|
"/run/current-system/sw"
|
|
];
|
|
environment = import ./environment.nix { inherit lib config pkgs; };
|
|
serviceConfig = {
|
|
ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --config /etc/hyperhive/serve.json";
|
|
SyslogIdentifier = "hive-c0re";
|
|
# Migrate hive-c0re's *own* state to the service user after an
|
|
# upgrade from a root-run install (systemd's StateDirectory only
|
|
# chowns the top-level dir, not pre-existing files inside it). The
|
|
# `+` prefix runs as root despite User = hive-core; `-` tolerates
|
|
# failure. coreutils ships `chown` but no `sh`, so invoke the
|
|
# binaries directly rather than through a shell.
|
|
#
|
|
# CRITICAL: exclude the per-agent `agents/` subtree. Its contents
|
|
# (each agent's `claude/` OAuth creds, `state/`, `harness/`,
|
|
# `config/`) are owned by the per-agent / manager users, and each
|
|
# container's `hive-agent-user-migrate` activation script chowns
|
|
# them back to that user on boot. Blanket-chowning them to hive-core
|
|
# makes every agent's `~/.claude` unreadable — logging them all out
|
|
# with no way to log back in. So chown everything *except* agents/,
|
|
# plus the `agents/` dir node itself (not its contents) so c0re can
|
|
# still create new per-agent subdirs.
|
|
ExecStartPre = [
|
|
# Install the safe.directory gitconfig at $HOME/.gitconfig
|
|
# (HOME = /var/lib/hyperhive) so c0re's `git fetch`/`rev-parse`
|
|
# against the agent-owned proposed repos pass the ownership guard.
|
|
# Placed before the chown below so it's chowned to hive-core too.
|
|
"+-${pkgs.coreutils}/bin/cp ${safeDirGitconfig} /var/lib/hyperhive/.gitconfig"
|
|
"+-${pkgs.findutils}/bin/find /var/lib/hyperhive -mindepth 1 -maxdepth 1 -not -name agents -exec ${pkgs.coreutils}/bin/chown -R hive-core:hive-core {} +"
|
|
"+-${pkgs.coreutils}/bin/chown hive-core:hive-core /var/lib/hyperhive/agents"
|
|
];
|
|
Restart = "on-failure";
|
|
RestartSec = 2;
|
|
User = "hive-core";
|
|
Group = "hive-core";
|
|
RuntimeDirectory = "hyperhive";
|
|
RuntimeDirectoryMode = runtimeDirMode;
|
|
RuntimeDirectoryPreserve = "yes";
|
|
StateDirectory = "hyperhive";
|
|
StateDirectoryMode = "0750";
|
|
# No OTEL credential here. hive-c0re's container-resource exporter
|
|
# targets this hive's own collector, which takes unauthenticated OTLP
|
|
# on the bridge; the upstream header belongs to the swarm tier
|
|
# (`swarm-otel.nix`), the only hop that leaves the swarm. Handing it to
|
|
# the daemon as well would put a secret on a process that has nowhere
|
|
# to present it.
|
|
LoadCredential =
|
|
# The secret store's client identity. `LoadCredential` and not a
|
|
# copy: root reads it at unit start and hive-core sees it 0400
|
|
# under `%d`, so it never gains a second on-disk copy. And
|
|
# ./glue-bao-tls.nix mints the key `0600` inside a `0700`
|
|
# directory, so hive-core cannot read the original at all.
|
|
# Loading it as a credential is what makes the store reachable
|
|
# from an unprivileged daemon without widening either.
|
|
# The gate is the identity, never `deploy.bao.enable`: a hive that
|
|
# reads a store on another machine holds one of these and runs no
|
|
# store. ./glue-queue-agent-credential.nix gates its own reader the
|
|
# same way.
|
|
lib.optionals baoClientIdentity [
|
|
"bao-client.pem:${baoDeploy.clientCertFile}"
|
|
"bao-client-key.pem:${baoDeploy.clientKeyFile}"
|
|
]
|
|
++ lib.optional (
|
|
baoClientIdentity && baoDeploy.serverCaFile != null
|
|
) "bao-ca.pem:${baoDeploy.serverCaFile}";
|
|
# Sandboxing. hive-c0re is unprivileged (runs as hive-core, never
|
|
# setuid), makes HTTP requests to forge/matrix/Anthropic (keeps INET),
|
|
# and delegates all privileged ops to hive-priv via a Unix socket.
|
|
# These directives deny the subset of kernel capabilities it
|
|
# provably doesn't need without restricting its network or
|
|
# filesystem access (RestrictAddressFamilies deferred — needs a
|
|
# watched deploy to verify no AF_UNIX/AF_INET gaps in socket paths).
|
|
NoNewPrivileges = true; # already runs as unprivileged user
|
|
PrivateTmp = true; # uses StateDirectory for tmpfiles, not /tmp
|
|
ProtectHome = true; # HOME = /var/lib/hyperhive; no /home/* access needed
|
|
# "strict" makes the entire filesystem read-only except for
|
|
# StateDirectory (/var/lib/hyperhive), RuntimeDirectory
|
|
# (/run/hyperhive), and ReadWritePaths below, which systemd keeps
|
|
# writable. Beyond the managed directories:
|
|
# - nix is invoked directly (lifecycle, meta, flake_check), but
|
|
# NIX_REMOTE=daemon routes all store writes through the host
|
|
# daemon — hive-c0re never writes to /nix itself.
|
|
# - flake.lock ops land in the meta worktree under StateDirectory
|
|
# (kept writable by systemd).
|
|
# - nix build worktrees live in PrivateTmp, not /tmp.
|
|
# - /etc writes (bind-mount edits) go through hive-priv via the
|
|
# privileged socket; /etc/hyperhive/serve.json is read-only.
|
|
# - the gateway's nginx include lives under
|
|
# /var/lib/hive-gateway/conf, deliberately outside StateDirectory
|
|
# (see hive-c0re/src/paths.rs::GATEWAY_CONF_DIR) so nginx never
|
|
# needs access to the rest of c0re's state — that separation
|
|
# means c0re itself needs an explicit carve-out to write there.
|
|
ProtectSystem = "strict";
|
|
ReadWritePaths = [ "/var/lib/hive-gateway/conf" ];
|
|
ProtectKernelTunables = true; # no sysctl writes
|
|
ProtectKernelLogs = true; # no /dev/kmsg access needed
|
|
ProtectControlGroups = true; # cgroup writes go through hive-priv, not c0re directly
|
|
RestrictNamespaces = true; # namespace creation goes through hive-priv
|
|
LockPersonality = true; # no personality changes needed
|
|
RestrictRealtime = true; # no real-time scheduling
|
|
};
|
|
};
|
|
|
|
# Socket unit for the hive-c0re admin socket. systemd creates and holds
|
|
# `/run/hyperhive/host.sock` before hive-c0re starts, then passes the fd
|
|
# via LISTEN_FDS (socket activation). Benefits: `hivectl` can connect
|
|
# the moment the socket unit is active — no racy retry window — and a
|
|
# hive-c0re restart never drops the socket inode, so queued commands
|
|
# drain cleanly.
|
|
#
|
|
# `hive-c0re serve` reads LISTEN_FDS via the `listenfd` crate and
|
|
# accepts the fd in preference to its own `bind()` path. When invoked
|
|
# directly (dev, CI, without the socket unit) LISTEN_FDS is absent and
|
|
# the traditional bind path runs unchanged — no regression.
|
|
systemd.sockets.hive-c0re = {
|
|
description = "hive-c0re admin socket";
|
|
wantedBy = [ "sockets.target" ];
|
|
socketConfig = {
|
|
# Must match the `--socket` arg passed to `hive-c0re serve`.
|
|
ListenStream = "/run/hyperhive/host.sock";
|
|
# `0660 root:hive-admin` — group-owned by `hive-admin` so operators in
|
|
# that group (services.hyperhive.c0re.adminUsers) drive `hivectl`
|
|
# without root; an empty adminUsers list leaves the group memberless,
|
|
# so it stays effectively root-only. See docs/trust-boundary/boundary.md.
|
|
SocketMode = "0660";
|
|
SocketGroup = "hive-admin";
|
|
# Only consulted when the directory is absent at socket-unit
|
|
# activation — i.e. when this unit wins the race with the service.
|
|
DirectoryMode = runtimeDirMode;
|
|
};
|
|
};
|
|
};
|
|
}
|