Watch
0
0
Fork
You've already forked hyperhive
0

hand agents the dashboard port, pinned claude and otel labels at container start

The three values that differ per host (deploy.hive-controller.dashboardPort,
claudeCodePackage and otel.extraResourceAttributes) no longer go into the
meta flake. hive-priv writes them into the container's
/etc/hyperhive-hive-facts with the hive name and domain, and the agent
reads them at boot:

- HIVE_DASHBOARD_PORT reaches every unit through the hive-facts generator;
  the web UI still defaults to 7000 without it.
- `claude` on PATH is a wrapper that runs HYPERHIVE_CLAUDE_CODE_PATH's
  claude when it is there, and the container's own claude-code otherwise.
  The host's serve.json keeps gc-rooting the pinned build.
- OTEL_RESOURCE_ATTRIBUTES moves out of the units' Environment= into a
  boot-time generator, which appends HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES.

The agent options services.hyperhive.agent.claudeCodePath (now a removed
option) and otel.extraResourceAttributes go; only the meta render set them.
render_flake takes no HiveEnv any more, so the memory-cap render test and
render_hive_flake_at, which only existed to vary HiveEnv into the render,
go with it.

Refs #4857
This commit is contained in:
atlas 2026-10-21 23:47:35 +02:00 • committed by mara
commit 9113651c10
23 changed files with 408 additions and 335 deletions

View file

@ -28,6 +28,28 @@ let
which the hive writes before every start, so no agent build sets it.
Remove the setting.
'';
claudeCodePathRemoved = ''
An agent reads the claude-code its hive pins at runtime, from
HYPERHIVE_CLAUDE_CODE_PATH in /etc/hyperhive-hive-facts, so no agent build
sets it. Pin it host-side with
services.hyperhive.deploy.hive-controller.claudeCodePackage.
'';
# `claude` on PATH. A hive that pins a claude-code
# (`services.hyperhive.deploy.hive-controller.claudeCodePackage`) hands its
# store path over at container start as `HYPERHIVE_CLAUDE_CODE_PATH`, and
# the host's system closure keeps it alive: nothing in this container
# references it. With no pin, or a path that is not there, this container's
# own claude-code runs. `exec -a claude` keeps the bare name in argv[0],
# which is how the harness finds its claude child (`find_claude_child`).
ownClaude = import ../claude-code-no-alsa.nix lib pkgs.claude-code;
claude = pkgs.writeShellScriptBin "claude" ''
pinned=''${HYPERHIVE_CLAUDE_CODE_PATH:-}
if [ -n "$pinned" ] && [ -x "$pinned/bin/claude" ]; then
exec -a claude "$pinned/bin/claude" "$@"
fi
exec -a claude ${ownClaude}/bin/claude "$@"
'';
in
{
imports = [
@ -68,6 +90,12 @@ in
"agent"
"claudeMemoryMaxBytes"
] memoryMaxBytesRemoved)
(lib.mkRemovedOptionModule [
"services"
"hyperhive"
"agent"
"claudeCodePath"
] claudeCodePathRemoved)
];
options.services.hyperhive.agent.icon = lib.mkOption {
@ -87,33 +115,6 @@ in
'';
};
options.services.hyperhive.agent.claudeCodePath = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/nix/store/…-claude-code-2.1.220";
description = ''
Store path of the `claude-code` this agent runs, or `null` (the
default) to use the `claude-code` from the container's own
nixpkgs.
Set by the generated meta flake when the operator sets
`services.hyperhive.deploy.hive-controller.claudeCodePackage` host-side, so a hive
can run a claude built from a *different* nixpkgs than the one
its agents evaluate against — a release channel can trail
unstable by weeks on this one package. It arrives as a path and
not a package because agents share the host's `/nix/store`, so
the build is already reachable here with its whole closure and
has nothing to travel.
When set, `claude` on PATH is a symlink to this path's
`bin/claude` and the container's own `claude-code` is dropped, so
there is only ever one claude in the container. Note that neither
the symlink nor anything else in the container's closure *refers*
to the target — keeping it alive is the host's job, see
`services.hyperhive.deploy.hive-controller.claudeCodePackage`.
'';
};
config = {
assertions = [
# Guard the inputs-routed-as-output pattern: the agent flake.nix is
@ -240,28 +241,7 @@ in
config.services.hyperhive.agent.packages.hive-metric
]
++ [
(
if config.services.hyperhive.agent.claudeCodePath == null then
import ../claude-code-no-alsa.nix lib pkgs.claude-code
else
# Host-pinned claude: a symlink farm around a path the
# container was handed as text. It has to be a derivation —
# `environment.systemPackages` coerces a store-path *string*
# with `toDerivation`, i.e. `builtins.storePath`, which pure
# evaluation rejects (`systemd.services.*.path` does the same,
# which is why the harness gets this via PATH like everything
# else rather than a unit-level entry). Interpolating the path
# into the builder is just text, so it evaluates anywhere.
#
# The symlink registers no store reference — the target isn't
# among this derivation's inputs, so nothing here keeps the
# binary alive. That is deliberate and it is the host's job:
# see `services.hyperhive.deploy.hive-controller.claudeCodePackage`.
pkgs.runCommandLocal "claude-code-pinned" { } ''
mkdir -p "$out/bin"
ln -s ${config.services.hyperhive.agent.claudeCodePath}/bin/claude "$out/bin/claude"
''
)
claude
]
++ (with pkgs; [
bashInteractive

View file

@ -2,8 +2,17 @@
# agent's closure does not carry it.
#
# hive-priv writes `/etc/hyperhive-hive-facts` into the container before every
# start (`HYPERHIVE_HIVE_NAME`, `HYPERHIVE_HIVE_DOMAIN`, one `KEY=value` line
# each). Two readers hand those lines on:
# start, one `KEY=value` line each:
#
# - `HYPERHIVE_HIVE_NAME`, `HYPERHIVE_HIVE_DOMAIN`;
# - `HIVE_DASHBOARD_PORT`, read by the harness's web UI (7000 when absent);
# - `HYPERHIVE_CLAUDE_CODE_PATH`, the claude-code the hive pins (`claude` in
# ./default.nix runs this container's own when absent);
# - `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`, appended to
# `OTEL_RESOURCE_ATTRIBUTES` by ./otel.nix (no extra labels when absent).
#
# Any of them may be absent: a hive-c0re older than the agent's build writes
# fewer. Two readers hand the lines on:
#
# - an environment generator, for the container's systemd: it passes them to
# every unit it starts, `machinectl shell` sessions (`hivectl choom`)
@ -23,8 +32,8 @@
let
userName = config.services.hyperhive.agent.user.name;
# Shell builtins only: nothing is assumed on PATH this early. Only the two
# known keys are passed on. `|| [ -n "$key" ]` keeps a last line that has
# Shell builtins only: nothing is assumed on PATH this early. Only the known
# keys are passed on. `|| [ -n "$key" ]` keeps a last line that has
# no trailing newline.
generator = pkgs.writeShellScript "hyperhive-hive-facts" ''
hive=unknown
@ -32,7 +41,8 @@ let
while IFS='=' read -r key value || [ -n "$key" ]; do
case "$key" in
HYPERHIVE_HIVE_NAME) hive=$value ;;
HYPERHIVE_HIVE_DOMAIN) ;;
HYPERHIVE_HIVE_DOMAIN | HIVE_DASHBOARD_PORT) ;;
HYPERHIVE_CLAUDE_CODE_PATH | HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES) ;;
*) continue ;;
esac
printf '%s=%s\n' "$key" "$value"

View file

@ -42,9 +42,17 @@ let
# agent, and their metric names already tell them apart. No `hive`: the
# swarm collector stamps it from the receiver that accepted the hive's
# push (nix/host-modules/swarm-otel.nix), so the build names no hive.
resourceAttributes =
"service.name=hyperhive-agent,agent=${userName},swarm=${swarmDisplayName}"
+ lib.optionalString (cfg.extraResourceAttributes != "") ",${cfg.extraResourceAttributes}";
resourceAttributes = "service.name=hyperhive-agent,agent=${userName},swarm=${swarmDisplayName}";
# Prints `OTEL_RESOURCE_ATTRIBUTES`: the labels above, then the hive's own
# (`HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`), which differ per host and so
# arrive at container start. That variable comes from ./hive-facts.nix's
# generator, which runs before this one and is absent when the hive sets no
# labels.
resourceGenerator = pkgs.writeShellScript "hyperhive-otel-resource" ''
printf 'OTEL_RESOURCE_ATTRIBUTES=%s%s\n' ${lib.escapeShellArg resourceAttributes} \
"''${HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES:+,$HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES}"
'';
# The variables an OTEL SDK reads on its own, in any language, in any
# process here.
@ -62,7 +70,6 @@ let
# which Prometheus/Mimir-family backends (incl. grafana-lgtm)
# silently drop without a deltatocumulative processor.
OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE = "cumulative";
OTEL_RESOURCE_ATTRIBUTES = resourceAttributes;
};
# The exporter's NAME picks the wire protocol, so `protocol` has to be
@ -118,19 +125,6 @@ in
'';
};
extraResourceAttributes = lib.mkOption {
type = lib.types.str;
default = "";
internal = true;
description = ''
Extra comma-separated entries appended to
`OTEL_RESOURCE_ATTRIBUTES` after the built-in
`service.name` / `agent` / `swarm` labels.
Host-driven via
`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`.
'';
};
debug = lib.mkOption {
type = lib.types.bool;
default = false;
@ -167,6 +161,14 @@ in
systemd.globalEnvironment = producerEnv;
environment.variables = producerEnv;
# `OTEL_RESOURCE_ATTRIBUTES` is in neither: a unit's own `Environment=`
# would override what the generator computes at boot. Units get it from
# the manager's environment; login shells run the same generator after
# ./hive-facts.nix's snippet has exported the hive's labels.
environment.etc."systemd/system-environment-generators/60-hyperhive-otel-resource".source =
resourceGenerator;
environment.extraInit = lib.mkAfter ''export "$(${resourceGenerator})"'';
# The journal this container writes is reachable from inside it and
# from nowhere else, which is why the forwarder runs here rather than
# the hive reading the container's journal from outside: the host-side

View file

@ -110,9 +110,9 @@ let
# in whole rather than interpolated — that is also the ONLY thing
# gc-rooting it: the resulting string carries store context, so this
# /etc entry genuinely references the package and the host's system
# closure holds it alive. Nothing container-side can — meta.rs writes
# the path into each agent's generated flake as a plain string
# literal, and text is not a reference. Hence the assertion below: do
# closure holds it alive. Nothing container-side can: hive-c0re hands
# the path to each container as text (`HYPERHIVE_CLAUDE_CODE_PATH`),
# and text is not a reference. Hence the assertion below: do
# NOT discard this context, and do not route the path through
# anything that drops it. The failure mode is a garbage-collected
# `claude` and a hive that can't take a turn, weeks after the commit

View file

@ -326,8 +326,9 @@
};
};
# What this hive hands its own agents through the meta-flake render. Per
# host, so two hives may set them differently.
# What this hive hands its own agents at container start, through
# `/etc/hyperhive-hive-facts`. Per host, so two hives may set them
# differently.
options.services.hyperhive.deploy.hive-controller = {
claudeCodePackage = lib.mkOption {
type = lib.types.nullOr lib.types.package;
@ -350,22 +351,22 @@
What travels into the container is the **store path**, not the
derivation: agents share the host's `/nix/store`, so the binary
and its full closure are already reachable there — nothing
needs rebuilding or copying. hive-c0re writes the path into
each agent's generated flake as a plain string literal (a bare
path fed to `lib.types.package` would run `builtins.storePath`,
which is illegal under pure evaluation) and the agent module
puts its `bin/` on the harness's PATH.
needs rebuilding or copying. hive-c0re hands the path to each
container at start as `HYPERHIVE_CLAUDE_CODE_PATH`, and the agent's
`claude` on PATH runs it, or the container's own `claude-code` when
the variable is absent or names nothing. A change reaches an agent
at its next container start, not through a rebuild.
The flip side of a plain string is that nothing in the agent's
own closure refers to it, so the container cannot keep it
alive. The **host** does that instead: this package is
interpolated into `/etc/hyperhive/serve.json`, which puts it in
the host's system closure — so it is gc-rooted by the running
generation for exactly as long as that generation is the one
the agents were rendered from. The cost is that
`nix-collect-garbage` cannot reclaim an old `claude-code` until
every agent has been rebuilt past it and the old generations
are gone.
the host's system closure, so it is gc-rooted by the running
generation. ⚠️ An agent started under an older generation keeps
that generation's path until its next start, and only the old
generation roots it: collect old generations after restarting
the agents, or an agent still running the old path falls back to
its own `claude-code` at its next `claude` spawn.
The swarm's `swarm-claude-login` and `swarm-review-classifier`
containers, where this host runs them, follow the same pin and
@ -377,7 +378,11 @@
dashboardPort = lib.mkOption {
type = lib.types.port;
default = 7000;
description = "TCP port this hive's hive-c0re dashboard listens on.";
description = ''
TCP port this hive's hive-c0re dashboard listens on. Agents get it
at container start as `HIVE_DASHBOARD_PORT`, for their web UI's
links back to the dashboard.
'';
};
};
}

View file

@ -236,7 +236,7 @@ in
};
# Read by this hive's hive-c0re, which stamps it on its own export and
# renders it into its agents', so it is a per-host choice.
# hands it to its agents at container start, so it is a per-host choice.
options.services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes = lib.mkOption {
type = lib.types.str;
default = "";

View file

@ -123,6 +123,25 @@ let
&& lib.all (e: s.exporters.${e}.endpoint == endpoint) s.service.pipelines.logs.exporters
&& !(agentNoOtel.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT);
}
{
# The hive's extra labels arrive at container start, so the build
# cannot hold the whole `OTEL_RESOURCE_ATTRIBUTES`: a boot-time
# generator computes it, and a unit-level `Environment=` copy would
# override that. The endpoint beside it is the presence control for the
# two absence lookups; `agentNoOtel` is the control for the generator.
name = "an agent's OTEL resource labels come from a boot-time generator, not the build";
ok =
let
generator =
m: m.environment.etc ? "systemd/system-environment-generators/60-hyperhive-otel-resource";
in
agentDefault.systemd.globalEnvironment ? OTEL_EXPORTER_OTLP_ENDPOINT
&& !(agentDefault.systemd.globalEnvironment ? OTEL_RESOURCE_ATTRIBUTES)
&& agentDefault.environment.variables ? OTEL_EXPORTER_OTLP_ENDPOINT
&& !(agentDefault.environment.variables ? OTEL_RESOURCE_ATTRIBUTES)
&& generator agentDefault
&& !(generator agentNoOtel);
}
{
# Presence control for the two cases above: with the switch off there
# is no collector in the container at all, so their passing is about

View file

@ -282,7 +282,7 @@ let
{
# The CLI is the one the agents run, not the container's own nixpkgs'.
# Unpinned is the control: it falls back to the container's own `claude-code`.
# serve.json is what hive-c0re renders every agent's `claudeCodePath` from.
# serve.json is what hive-c0re hands every agent's `HYPERHIVE_CLAUDE_CODE_PATH` from.
name = "the agents, the claude login minter and the review classifier run the claude-code this host pins";
ok =
let

View file

@ -1,7 +1,9 @@
# `checks.script-test-agent-hive-facts` — runs the two readers of
# `/etc/hyperhive-hive-facts` in ../agent-modules/hive-facts.nix: the systemd
# environment generator and the login-shell snippet `/etc/set-environment`
# sources. The cases are in ./agent-hive-facts.sh.
# sources. Also the two consumers of what they pass on: ../agent-modules/otel.nix's
# `OTEL_RESOURCE_ATTRIBUTES` generator and ../agent-modules/default.nix's `claude`.
# The cases are in ./agent-hive-facts.sh.
#
# What runs is the rendered generator and snippet. The one edit is the facts
# file's path, moved under the build directory, and the snippet's reference to
@ -26,6 +28,11 @@ let
;
machine = agentWith { services.hyperhive.agent.user.name = "a1"; };
# As text: building it would build the claude-code it falls back to.
claude = lib.findFirst (
p: (p.name or "") == "claude"
) (throw "no `claude` in systemPackages") machine.environment.systemPackages;
in
pkgs.runCommand "hyperhive-script-test-agent-hive-facts"
{
@ -33,6 +40,9 @@ pkgs.runCommand "hyperhive-script-test-agent-hive-facts"
machine.environment.etc."systemd/system-environment-generators/50-hyperhive-hive-facts".source;
PROFILE = machine.environment.etc."hyperhive/hive-facts.sh".source;
EXTRA_INIT = machine.environment.extraInit;
OTEL_GENERATOR =
machine.environment.etc."systemd/system-environment-generators/60-hyperhive-otel-resource".source;
CLAUDE_WRAPPER = builtins.unsafeDiscardStringContext claude.text;
SHELLS = "${pkgs.dash}/bin/dash ${pkgs.bash}/bin/sh";
}
''

View file

@ -40,7 +40,7 @@ check() {
for sh in $SHELLS; do
# Only what the snippet exported reaches the child's environment. No PATH:
# the snippet must not need one.
got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort)
got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|HIVE_DASHBOARD_PORT=|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort)
if [ "$got" != "$(printf '%s\n' "$want" | sort)" ]; then
fail "$sh: a child of the sourcing shell saw:"$'\n'"$got"
fi
@ -61,11 +61,69 @@ check "a last line without a newline is kept" "HYPERHIVE_HIVE_DOMAIN=pr1ma.examp
HYPERHIVE_HIVE_NAME=pr1ma
CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=pr1ma-a1"
printf 'HIVE_DASHBOARD_PORT=7123\nHYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code\nHYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\n' >"$facts"
check "the per-host values, a value holding = kept whole" "HIVE_DASHBOARD_PORT=7123
HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code
HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod
CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=unknown-a1"
: >"$facts"
check "an empty file (no facts)" "CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=unknown-a1"
rm -f "$facts"
check "no file" "CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=unknown-a1"
# OTEL_RESOURCE_ATTRIBUTES, for units (the generator, run with what the
# hive-facts generator printed) and for login shells (the extraInit line, after
# the snippet). The base labels come from the build; the hive's are appended.
base="service.name=hyperhive-agent,agent=a1,swarm=unknown"
otel_check() {
case=$1
count=$((count + 1))
local want="OTEL_RESOURCE_ATTRIBUTES=$2" got sh passed
mapfile -t passed < <(env -i "$dir/generator")
got=$(env -i "${passed[@]}" "$OTEL_GENERATOR")
if [ "$got" != "$want" ]; then fail "generator printed: $got, wanted: $want"; fi
for sh in $SHELLS; do
got=$(env -i "$sh" -c ". '$dir/profile'; $otel_init; exec '$envbin'" | grep '^OTEL_RESOURCE_ATTRIBUTES=')
if [ "$got" != "$want" ]; then fail "$sh: a login shell saw: $got"; fi
done
}
case="otel setup"
otel_init=$(printf '%s\n' "$EXTRA_INIT" | grep -F "$OTEL_GENERATOR")
if [ -z "$otel_init" ]; then fail "/etc/set-environment does not run the OTEL resource generator"; fi
if [ "$(printf '%s\n' "$EXTRA_INIT" | grep -n . | grep -F "$OTEL_GENERATOR" | cut -d: -f1)" -lt \
"$(printf '%s\n' "$EXTRA_INIT" | grep -nxF '. /etc/hyperhive/hive-facts.sh' | cut -d: -f1)" ]; then
fail "/etc/set-environment runs the OTEL resource generator before the hive-facts snippet"
fi
printf 'HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\n' >"$facts"
otel_check "the hive's labels are appended" "$base,team=a,env=prod"
rm -f "$facts"
otel_check "no hive labels: the base alone, no trailing comma" "$base"
# `claude`: the hive's pinned build when it is there, this container's own
# otherwise. Both builds are stand-ins that print which one ran.
mkdir -p "$dir/own/bin" "$dir/pinned/bin"
printf '#!%s\necho own "$@"\n' "$BASH" >"$dir/own/bin/claude"
printf '#!%s\necho pinned "$@"\n' "$BASH" >"$dir/pinned/bin/claude"
chmod +x "$dir/own/bin/claude" "$dir/pinned/bin/claude"
printf '%s' "$CLAUDE_WRAPPER" | sed -E "s|/nix/store/[^ ]+/bin/claude|$dir/own/bin/claude|" >"$dir/claude"
case="claude setup"
if ! grep -qF "$dir/own/bin/claude" "$dir/claude"; then fail "the rewrite did not reach the fallback claude"; fi
claude_check() {
case=$1
count=$((count + 1))
local got pin=()
if [ -n "$2" ]; then pin=("HYPERHIVE_CLAUDE_CODE_PATH=$2"); fi
# Under this shell: the shebang names the build's bash, not in this sandbox.
got=$(env -i "${pin[@]}" "$BASH" "$dir/claude" -p hi)
if [ "$got" != "$3 -p hi" ]; then fail "ran: $got, wanted: $3 -p hi"; fi
}
claude_check "the pinned claude when the hive hands one over" "$dir/pinned" pinned
claude_check "the container's own claude with no pin" "" own
claude_check "the container's own claude when the pinned path is gone" "$dir/gone" own
echo "$count cases, $failures failed"
[ "$failures" -eq 0 ]