diff --git a/docs/agent-lifecycle/config-changes.md b/docs/agent-lifecycle/config-changes.md index ac36494a..7f914a8f 100644 --- a/docs/agent-lifecycle/config-changes.md +++ b/docs/agent-lifecycle/config-changes.md @@ -119,7 +119,8 @@ declares one flake input per agent (`agent-.url = `nixosConfigurations.` output per agent. Each output wraps `inputs.agent-.nixosModules.default` with the identity + `HIVE_PORT` / `HIVE_LABEL` injection module. Values that differ per host -(the dashboard port, the pinned claude-code, extra OTEL labels) aren't in it: +(the dashboard port, the pinned claude-code, extra OTEL labels, the queue +address) aren't in it: `hive-priv` writes them to the container's `/etc/hyperhive-hive-facts` at start (see `docs/process/conventions.md`). Containers run against `--flake /var/lib/hyperhive/meta#`. diff --git a/docs/process/conventions.md b/docs/process/conventions.md index 2ab12c09..9fe7b8ee 100644 --- a/docs/process/conventions.md +++ b/docs/process/conventions.md @@ -41,8 +41,9 @@ Four env vars cover the identity surface, read by Inside an agent container, the hive's name and domain don't come from the agent's build, and neither do the hive's other per-host values: the dashboard port (`HIVE_DASHBOARD_PORT`), the pinned claude-code -(`HYPERHIVE_CLAUDE_CODE_PATH`) and extra OTEL labels -(`HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`). `hive-priv` writes them to +(`HYPERHIVE_CLAUDE_CODE_PATH`), extra OTEL labels +(`HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`) and the queue's address +(`HIVE_AGENT_NATS_URL`, from `deploy.hive-controller.queue.agentNatsUrl`). `hive-priv` writes them to `/etc/hyperhive-hive-facts` before every container start, leaving out any the hive doesn't set; the agent then uses its own default. The agent's environment generator (`nix/agent-modules/hive-facts.nix`) passes them to every unit, diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 86055ea6..0d3376e2 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -176,7 +176,9 @@ holds it in memory only. The one thing to configure is where the queue is: | ------------------------------------------- | ------------------------------------------------------------- | | `deploy.hive-controller.queue.agentNatsUrl` | where the queue listens, as an agent **container** reaches it | -On every hive, `agentNatsUrl` defaults to the same +It reaches each container at start, as `HIVE_AGENT_NATS_URL` in +`/etc/hyperhive-hive-facts`, so no agent build carries it. On every hive, +`agentNatsUrl` defaults to the same `tls://:` as the hive's own. On the queue's host the name resolves inside a container to the bridge address, where the firewall opens the port; on any other hive it resolves through the host's DNS, diff --git a/hive-c0re/src/lifecycle/mod.rs b/hive-c0re/src/lifecycle/mod.rs index b70b440b..579e29ec 100644 --- a/hive-c0re/src/lifecycle/mod.rs +++ b/hive-c0re/src/lifecycle/mod.rs @@ -170,8 +170,8 @@ pub fn network_isolation_from_env() -> Result } /// The hive's facts every container is handed at runtime, so that no agent -/// build carries a value that differs between hosts. The name, domain and -/// OTEL resource labels come from hive-c0re's own unit environment +/// build carries a value that differs between hosts. The name, domain, OTEL +/// resource labels and queue address come from hive-c0re's own unit environment /// (`nix/host-modules/hive-c0re/environment.nix`), where an unset or empty /// variable is `None`; the dashboard port and pinned claude from `hive`. fn hive_facts(hive: &HiveEnv) -> hive_priv_sock::HiveFacts { @@ -190,6 +190,7 @@ fn hive_facts_from( dashboard_port: Some(hive.dashboard_port), claude_code_path: hive.claude_code_path.clone(), otel_extra_resource_attributes: read("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES"), + agent_nats_url: read("HIVE_AGENT_NATS_URL"), } } diff --git a/hive-c0re/src/lifecycle/tests.rs b/hive-c0re/src/lifecycle/tests.rs index b68ef27e..d638cdb2 100644 --- a/hive-c0re/src/lifecycle/tests.rs +++ b/hive-c0re/src/lifecycle/tests.rs @@ -528,9 +528,9 @@ async fn run_with_deadline_returns_output_of_a_fast_child() { assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi"); } -/// The three per-host values reach the container as facts: the dashboard -/// port and pinned claude from the hive's config, the OTEL labels from the -/// daemon's environment. +/// The per-host values reach the container as facts: the dashboard port and +/// pinned claude from the hive's config, the OTEL labels and the queue +/// address from the daemon's environment. #[test] fn hive_facts_carry_the_per_host_values() { let hive = HiveEnv { @@ -538,8 +538,10 @@ fn hive_facts_carry_the_per_host_values() { claude_code_path: Some("/nix/store/0abc-claude-code".to_owned()), ..HiveEnv::default() }; - let facts = hive_facts_from(&hive, |name| { - (name == "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES").then(|| "team=a".to_owned()) + let facts = hive_facts_from(&hive, |name| match name { + "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES" => Some("team=a".to_owned()), + "HIVE_AGENT_NATS_URL" => Some("tls://nats.t.local:4222".to_owned()), + _ => None, }); assert_eq!(facts.dashboard_port, Some(7123)); assert_eq!( @@ -550,6 +552,10 @@ fn hive_facts_carry_the_per_host_values() { facts.otel_extra_resource_attributes.as_deref(), Some("team=a") ); + assert_eq!( + facts.agent_nats_url.as_deref(), + Some("tls://nats.t.local:4222") + ); } /// Unset and empty both leave a fact out, so the agent falls back to its @@ -561,5 +567,6 @@ fn hive_facts_leave_unset_values_out() { }); assert_eq!(facts.claude_code_path, None); assert_eq!(facts.otel_extra_resource_attributes, None); + assert_eq!(facts.agent_nats_url, None); assert_eq!(facts.name, None); } diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index ee864626..77b907fd 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -1175,12 +1175,12 @@ mod tests { ); } - /// The queue address reaches the harness as env only. The agent module - /// declares no `services.hyperhive.agent.queue.*` option, so rendering one - /// into the meta flake fails every agent's evaluation. + /// The queue address reaches the container in its hive facts, so the render + /// carries neither it nor any `services.hyperhive.agent.queue.*` option, + /// which no agent module declares. #[test] - fn queue_address_is_forwarded_as_env_and_no_option() { - assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_NATS_URL")); + fn queue_address_is_not_rendered_into_the_flake() { + assert!(!FORWARDED_VARS.contains(&"HIVE_AGENT_NATS_URL")); assert!( !FORWARDED_VARS.contains(&"HIVE_AGENT_OIDC_TOKEN_ENDPOINT"), "no agent reads a token endpoint" diff --git a/hive-meta-flake/src/lib.rs b/hive-meta-flake/src/lib.rs index cf7f0861..86a1d53a 100644 --- a/hive-meta-flake/src/lib.rs +++ b/hive-meta-flake/src/lib.rs @@ -54,10 +54,6 @@ pub const CANONICAL_INPUTS: &[&str] = &["nixpkgs"]; /// names it has a non-empty value for, so absent options render no no-op /// `FOO = ""` lines. pub const FORWARDED_VARS: &[&str] = &[ - // Where the swarm queue is, as an agent container reaches it. An address - // the *host* computes, never a loopback address, which inside a container - // is the agent itself. - "HIVE_AGENT_NATS_URL", // Where the swarm secret store listens. The hive's own `BAO_ADDR` value, // forwarded under a separate name so the container never inherits the // hive's store environment wholesale — the address is the only part of it @@ -66,9 +62,8 @@ pub const FORWARDED_VARS: &[&str] = &[ // The mount path of the store's services `pki` engine, whose root an agent // fetches to trust the swarm. Forwarded with the address, under the same gate. "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", - // Where an agent's `swarm-logs` queries, computed by the host for the - // same reason as the queue's: it is a gateway address, and a container - // cannot derive it from anything it holds. + // Where an agent's `swarm-logs` queries, computed by the host: it is a + // gateway address, and a container cannot derive it from anything it holds. "HIVE_AGENT_LOGS_QUERY_URL", // The metrics store's origin, for `swarm-metrics`, for the same reason. "HIVE_AGENT_METRICS_URL", diff --git a/hive-priv-sock/src/lib.rs b/hive-priv-sock/src/lib.rs index 331c50d2..95bc80f7 100644 --- a/hive-priv-sock/src/lib.rs +++ b/hive-priv-sock/src/lib.rs @@ -196,6 +196,9 @@ pub struct HiveFacts { /// `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`: comma-separated `k=v` /// entries the agent appends to its `OTEL_RESOURCE_ATTRIBUTES`. pub otel_extra_resource_attributes: Option, + /// `HIVE_AGENT_NATS_URL`: where the agent reaches the swarm queue, from + /// `deploy.hive-controller.queue.agentNatsUrl`. + pub agent_nats_url: Option, } /// Wall-clock bound hive-priv puts on one [`PrivRequest::MeasureAgentDisk`] diff --git a/hive-priv/src/container_config.rs b/hive-priv/src/container_config.rs index 0fc3e297..2cf88691 100644 --- a/hive-priv/src/container_config.rs +++ b/hive-priv/src/container_config.rs @@ -402,6 +402,13 @@ fn render_hive_facts(facts: &HiveFacts) -> Result { 1024, &|b: u8| dns(b) || matches!(b, b'=' | b',' | b'/' | b':' | b'%' | b'+' | b'@'), ), + // `://:`. + ( + "HIVE_AGENT_NATS_URL", + facts.agent_nats_url.as_ref(), + 253, + &|b: u8| dns(b) || matches!(b, b':' | b'/'), + ), ] { let Some(value) = value else { continue }; if value.is_empty() || value.len() > max_len || !value.bytes().all(allowed) { @@ -774,8 +781,8 @@ mod tests { } } - /// The three per-host values agents used to get at build time arrive - /// here, one line each. + /// The per-host values agents used to get at build time arrive here, one + /// line each. #[test] fn hive_facts_render_the_per_host_values() { let facts = HiveFacts { @@ -784,13 +791,15 @@ mod tests { otel_extra_resource_attributes: Some( "deployment.environment=prod,team=a%2Fb".to_owned(), ), + agent_nats_url: Some("tls://nats.example.test:4222".to_owned()), ..HiveFacts::default() }; assert_eq!( render_hive_facts(&facts).unwrap(), "HIVE_DASHBOARD_PORT=7123\n\ HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code-2.1.220\n\ - HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=deployment.environment=prod,team=a%2Fb\n" + HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=deployment.environment=prod,team=a%2Fb\n\ + HIVE_AGENT_NATS_URL=tls://nats.example.test:4222\n" ); } diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index cafc97c9..d7f47540 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -36,7 +36,7 @@ let ''; queueRemoved = '' The harness reads the queue's address at runtime from HIVE_AGENT_NATS_URL, - which the hive forwards into every container, and authenticates with the + in /etc/hyperhive-hive-facts, and authenticates with the agent's own secret from the swarm secret store, so no agent build sets a queue option. Remove the setting. ''; diff --git a/nix/agent-modules/hive-facts.nix b/nix/agent-modules/hive-facts.nix index 864965bb..e9010ebf 100644 --- a/nix/agent-modules/hive-facts.nix +++ b/nix/agent-modules/hive-facts.nix @@ -10,6 +10,8 @@ # ./default.nix runs this container's own when absent); # - `HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES`, appended to # `OTEL_RESOURCE_ATTRIBUTES` by ./otel.nix (no extra labels when absent). +# - `HIVE_AGENT_NATS_URL`, where the harness reaches the swarm queue (it fails +# at boot when absent). # # Any of them may be absent: a hive-c0re older than the agent's build writes # fewer. Two readers hand the lines on: @@ -43,6 +45,7 @@ let HYPERHIVE_HIVE_NAME) hive=$value ;; HYPERHIVE_HIVE_DOMAIN | HIVE_DASHBOARD_PORT) ;; HYPERHIVE_CLAUDE_CODE_PATH | HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES) ;; + HIVE_AGENT_NATS_URL) ;; *) continue ;; esac printf '%s=%s\n' "$key" "$value" diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 5ea7e110..53b10636 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -222,8 +222,8 @@ in HIVE_C0RE_OIDC_CLIENT_ID = "hive-${config.services.hyperhive.hiveName}"; } // - # The agents' half of the same queue, forwarded by `hive_c0re::meta` into - # every container. + # Where the agents reach the queue: the address goes into every container in + # its hive facts (`hive_c0re::lifecycle::hive_facts`). # # ⚠️ By name, like `HIVE_C0RE_NATS_URL` above, and never a loopback # address: inside a container `127.0.0.1` is the agent itself. diff --git a/nix/script-tests/agent-hive-facts.sh b/nix/script-tests/agent-hive-facts.sh index 58f57ac7..2d1b3517 100644 --- a/nix/script-tests/agent-hive-facts.sh +++ b/nix/script-tests/agent-hive-facts.sh @@ -40,7 +40,7 @@ check() { for sh in $SHELLS; do # Only what the snippet exported reaches the child's environment. No PATH: # the snippet must not need one. - got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|HIVE_DASHBOARD_PORT=|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort) + got=$(env -i "$sh" -c ". '$dir/profile'; exec '$envbin'" | grep -E '^(HYPERHIVE_|HIVE_DASHBOARD_PORT=|HIVE_AGENT_NATS_URL=|CLAUDE_|LD_PRELOAD=|__hyperhive)' | sort) if [ "$got" != "$(printf '%s\n' "$want" | sort)" ]; then fail "$sh: a child of the sourcing shell saw:"$'\n'"$got" fi @@ -61,10 +61,11 @@ check "a last line without a newline is kept" "HYPERHIVE_HIVE_DOMAIN=pr1ma.examp HYPERHIVE_HIVE_NAME=pr1ma CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=pr1ma-a1" -printf 'HIVE_DASHBOARD_PORT=7123\nHYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code\nHYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\n' >"$facts" +printf 'HIVE_DASHBOARD_PORT=7123\nHYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code\nHYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod\nHIVE_AGENT_NATS_URL=tls://nats.example.test:4222\n' >"$facts" check "the per-host values, a value holding = kept whole" "HIVE_DASHBOARD_PORT=7123 HYPERHIVE_CLAUDE_CODE_PATH=/nix/store/0abc-claude-code HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES=team=a,env=prod +HIVE_AGENT_NATS_URL=tls://nats.example.test:4222 CLAUDE_REMOTE_CONTROL_SESSION_NAME_PREFIX=unknown-a1" : >"$facts"