From b8c5840299c33e239f41ee8b2bab78806c785d3e Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 01:36:52 +0200 Subject: [PATCH 01/10] swarm-controller: retry hive provisioning until the store is up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read policy and cert-auth role for each hive were written once, at startup. On the deploy that surfaced this, the store was still coming up, the pass logged its warning and moved on, and no hive could log in until someone restarted the daemon — while cert auth answered "no chain matching all constraints", which reads like a certificate problem rather than a role that was never created. The bootstrap unit in swarm-bao.nix lost the same race and won on its retry 30s later. A daemon that boots alongside its store loses that race routinely; on a normal boot it is the ordinary case. The two passes fold into one `provision()` that logs in once instead of twice for two loops over the same list, keeping policy before role since the role names the policy. `ensure_hive_access` still awaits the first pass, so a store that is already up leaves nothing deferred, and only a pass that could not reach the store at all spawns the retry. The retry is `config_pr::spawn`'s idiom from this same crate: an interval task whose first tick is immediate. Its cadence and bound match the bootstrap unit's — 30s, ~a day — because the two halves of one race should not disagree about how long a wait is worth. `Error::MissingEnv` is what keeps it from spinning forever: no `BAO_*` set means a deployment that runs no store, where asking again changes nothing, so it returns Ok. Everything else is retryable, including an authority file that is not placed yet — the unit that writes it starts alongside this one. Both cases previously landed in the same "not managed here" line, so a store that was late looked exactly like one that was never configured. Per-hive failures keep their old behaviour: logged, skipped, Ok. A store that refuses one hive's write refuses it again, so the next start really is the right retry for those, and the module doc still says so. Closes #4176. --- swarm-controller/src/main.rs | 10 +- swarm-controller/src/read_policy.rs | 252 +++++++++++++++++++++------- 2 files changed, 198 insertions(+), 64 deletions(-) diff --git a/swarm-controller/src/main.rs b/swarm-controller/src/main.rs index dbe17243..c0f2e034 100644 --- a/swarm-controller/src/main.rs +++ b/swarm-controller/src/main.rs @@ -1625,13 +1625,11 @@ async fn main() -> Result<()> { let hives = load_hives(); // Before serving, because a hive whose role does not exist cannot log in, // and one whose policy does not exist logs in able to read nothing — - // either way it cannot collect what this daemon writes for it. Policy - // first: the role names the policy, so writing it the other way round - // leaves a window where the role points at nothing. Same "log and carry - // on" shape as every connect above. + // either way it cannot collect what this daemon writes for it. A store + // that is not up yet is retried in the background instead of being + // abandoned, since the two of them boot together. let hive_names: Vec = hives.iter().map(|h| h.name.clone()).collect(); - read_policy::ensure_hive_policies(&hive_names).await; - read_policy::ensure_hive_roles(&hive_names).await; + read_policy::ensure_hive_access(hive_names).await; let state = AppState { hives: Arc::new(hives), diff --git a/swarm-controller/src/read_policy.rs b/swarm-controller/src/read_policy.rs index 70769cbb..af437575 100644 --- a/swarm-controller/src/read_policy.rs +++ b/swarm-controller/src/read_policy.rs @@ -6,44 +6,167 @@ //! without the policy the token it gets may read nothing — either way the read //! half of a delivery answers 403. //! -//! Both are startup passes rather than hooks. The policy document is the same -//! for every hive and depends on nothing ([`swarm_secret_client::policy`] -//! explains why it is that wide), and the hive list is loaded once because a -//! config change means a redeploy. Per-hive failures are reported and skipped — -//! one hive that cannot be provisioned should not take down a daemon that -//! serves everything else, and the next start retries it. +//! A pass runs at startup, and repeats until it reaches the store. The two +//! start together, so the store being the slower one is the ordinary case +//! rather than an edge one, and a pass that gave up there provisions nobody +//! until something restarts this daemon. Per-hive failures are different: they +//! are reported and skipped — one hive that cannot be provisioned should not +//! take down a daemon that serves everything else, and a store that answers is +//! answering the same way next time, so the next start is the retry for those. +//! +//! The policy document is the same for every hive and depends on nothing +//! ([`swarm_secret_client::policy`] explains why it is that wide), and the hive +//! list is loaded once because a config change means a redeploy. + +use std::{future::Future, time::Duration}; use anyhow::{Context, Result}; -use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, policy}; +use swarm_secret_client::{Error, SecretStore, client::DEFAULT_CERT_MOUNT, policy}; /// File holding the authority hives are issued from, as /// `swarm-controller.nix` names it. pub const ENV_HIVE_CLIENT_CA: &str = "SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE"; -/// Give every hive in `hives` the read policy its certificate will carry. +/// How long to wait before re-attempting a pass that never reached the store. /// -/// Does nothing when this deployment has no store to reach, which is the -/// shape a controller without a secret store runs in. -pub async fn ensure_hive_policies(hives: &[String]) { - if hives.is_empty() { - return; - } - let store = match crate::store::connect().await { - Ok(store) => store, - Err(e) => { - tracing::info!(reason = %e, "no secret store reachable; hive read policies are not managed here"); - return; - } - }; - for hive in hives { - if let Err(e) = write_one(&store, hive).await { - tracing::warn!(hive, error = %format!("{e:#}"), "writing this hive's read policy failed"); - } +/// `nix/host-modules/swarm-bao.nix`'s bootstrap unit retries the other side of +/// this same race on the same cadence: the two halves of one race should not +/// disagree about how long a wait is worth. +const RETRY_INTERVAL: Duration = Duration::from_secs(30); + +/// How many times [`RETRY_INTERVAL`] is waited before giving up. A store that +/// has not appeared within [`RETRY_WINDOW`] is absent rather than slow, and a +/// daemon logging into the void forever tells an operator nothing. +const RETRY_ATTEMPTS: u32 = 2_880; + +/// What [`RETRY_ATTEMPTS`] at [`RETRY_INTERVAL`] come to. A named third +/// constant because it is the claim the other two make together, and either +/// one of them can be edited alone. +const RETRY_WINDOW: Duration = Duration::from_hours(24); + +/// A pass that never got as far as the store, carrying what to say about it. +/// +/// Returned rather than logged where it happens: the first attempt deserves a +/// warning and the two-hundredth is noise, and only the caller knows which one +/// this is. +struct Unreachable(String); + +/// Give every hive in `hives` the access its certificate needs. +/// +/// Returns once the first pass is done, so a store that is already up leaves +/// nothing deferred. When that pass cannot reach the store, it repeats in the +/// background for up to [`RETRY_WINDOW`]. +pub async fn ensure_hive_access(hives: Vec) { + if let Err(Unreachable(reason)) = provision(&hives).await { + tracing::warn!( + reason, + retry_in = ?RETRY_INTERVAL, + "hive provisioning could not reach the swarm secret store; retrying in the background" + ); + spawn_retry(hives); } } +/// Repeat [`provision`] until it reaches the store, or [`RETRY_WINDOW`] passes. +fn spawn_retry(hives: Vec) { + tokio::spawn(async move { + let reached = retry_until_ok(RETRY_INTERVAL, RETRY_ATTEMPTS, move || { + let hives = hives.clone(); + async move { + let outcome = provision(&hives).await; + if let Err(Unreachable(reason)) = &outcome { + tracing::debug!(reason, "hive provisioning: store still unreachable"); + } + outcome + } + }) + .await; + if let Some(attempt) = reached { + tracing::info!(attempt, "hive provisioning reached the store"); + } else { + tracing::warn!( + attempts = RETRY_ATTEMPTS, + window = ?RETRY_WINDOW, + "giving up on hive provisioning; no hive can log in to the store until this daemon is restarted" + ); + } + }); +} + +/// Run `attempt` every `interval` until it succeeds, at most `attempts` times. +/// Answers with the attempt that succeeded, or `None` if none did. +/// +/// The cadence and the bound are parameters rather than the constants above so +/// that a test can drive this loop without waiting out a real day. +async fn retry_until_ok(interval: Duration, attempts: u32, mut attempt: F) -> Option +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut ticker = tokio::time::interval(interval); + // `interval`'s first tick is immediate, and that pass is the one the caller + // already ran before deciding to retry at all. + ticker.tick().await; + for n in 1..=attempts { + ticker.tick().await; + if attempt().await.is_ok() { + return Some(n); + } + } + None +} + +/// One pass: log in once, write every hive's policy, then every hive's role. +/// +/// Policy first, because the role names the policy — the other order leaves a +/// window in which the role points at nothing. +/// +/// # Errors +/// [`Unreachable`] for the states a later attempt can resolve: a store that is +/// not up, and an authority file that is not placed yet. A deployment that is +/// configured for neither is `Ok` — there is nothing to wait for. +async fn provision(hives: &[String]) -> Result<(), Unreachable> { + if hives.is_empty() { + return Ok(()); + } + let store = match crate::store::connect().await { + Ok(store) => store, + // An unset `BAO_*` variable is a deployment that runs no store, not + // one whose store is late: asking again changes nothing. + Err(Error::MissingEnv(var)) => { + tracing::info!( + var, + "no secret store configured; hive access is not managed here" + ); + return Ok(()); + } + Err(e) => return Err(Unreachable(e.to_string())), + }; + for hive in hives { + if let Err(e) = write_policy_for(&store, hive).await { + tracing::warn!(hive, error = %format!("{e:#}"), "writing this hive's read policy failed"); + } + } + + let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else { + tracing::info!("no hive certificate authority configured; hive roles are not managed here"); + return Ok(()); + }; + let path = path.to_string_lossy().into_owned(); + // Retryable rather than fatal: the unit that places this file and this + // daemon start together, so "not there yet" is a moment, not a verdict. + let ca = std::fs::read_to_string(&path) + .map_err(|e| Unreachable(format!("reading the hive authority {path}: {e}")))?; + for hive in hives { + if let Err(e) = write_role(&store, hive, &ca).await { + tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed"); + } + } + Ok(()) +} + /// One hive's policy: its own name, the shared document. -async fn write_one(store: &SecretStore, hive: &str) -> Result<()> { +async fn write_policy_for(store: &SecretStore, hive: &str) -> Result<()> { let name = policy::hive_object_name(hive)?; store .write_policy(&name, &policy::render()) @@ -53,38 +176,6 @@ async fn write_one(store: &SecretStore, hive: &str) -> Result<()> { Ok(()) } -/// Create each hive's cert-auth role, so the certificate it already holds logs -/// in and carries the policy written above. -/// -/// Does nothing when no authority is configured, which is the deployment that -/// has not onboarded a hive yet. -pub async fn ensure_hive_roles(hives: &[String]) { - let Some(path) = std::env::var_os(ENV_HIVE_CLIENT_CA) else { - tracing::info!("no hive certificate authority configured; hive roles are not managed here"); - return; - }; - let path = path.to_string_lossy().into_owned(); - let ca = match std::fs::read_to_string(&path) { - Ok(ca) => ca, - Err(e) => { - tracing::warn!(path, error = %e, var = ENV_HIVE_CLIENT_CA, "reading the hive authority failed; hive roles are not created"); - return; - } - }; - let store = match crate::store::connect().await { - Ok(store) => store, - Err(e) => { - tracing::warn!(error = %e, "connecting to the swarm secret store failed; hive roles are not created"); - return; - } - }; - for hive in hives { - if let Err(e) = write_role(&store, hive, &ca).await { - tracing::warn!(hive, error = %format!("{e:#}"), "creating this hive's store role failed"); - } - } -} - /// The role for one hive: its own certificate's subject, its own policy. async fn write_role(store: &SecretStore, hive: &str, ca: &str) -> Result<()> { let name = policy::hive_object_name(hive)?; @@ -97,3 +188,48 @@ async fn write_role(store: &SecretStore, hive: &str, ca: &str) -> Result<()> { tracing::info!(hive, role = %name, "hive store role in place"); Ok(()) } + +#[cfg(test)] +mod tests { + use std::{cell::Cell, time::Duration}; + + use super::{RETRY_ATTEMPTS, RETRY_INTERVAL, RETRY_WINDOW, Unreachable, retry_until_ok}; + + #[test] + fn the_retry_bound_is_the_window_it_claims() { + assert_eq!(RETRY_INTERVAL * RETRY_ATTEMPTS, RETRY_WINDOW); + } + + /// The property this issue is about: a pass that fails because the store is + /// not up yet happens again. + #[tokio::test] + async fn a_pass_is_repeated_until_it_reaches_the_store() { + let calls = Cell::new(0_u32); + let reached = retry_until_ok(Duration::from_millis(1), 10, || { + calls.set(calls.get() + 1); + let still_down = calls.get() < 3; + async move { + if still_down { + Err(Unreachable("store not up".to_owned())) + } else { + Ok(()) + } + } + }) + .await; + assert_eq!(reached, Some(3)); + assert_eq!(calls.get(), 3, "the loop stops at the first success"); + } + + #[tokio::test] + async fn the_loop_is_bounded_rather_than_endless() { + let calls = Cell::new(0_u32); + let reached = retry_until_ok(Duration::from_millis(1), 5, || { + calls.set(calls.get() + 1); + async { Err(Unreachable("store not up".to_owned())) } + }) + .await; + assert_eq!(reached, None, "a store that never appears is given up on"); + assert_eq!(calls.get(), 5, "one attempt per interval, and no more"); + } +} From 68711796efc9b0691906aa778cb371c298a072c9 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 02:14:08 +0200 Subject: [PATCH 02/10] otel: forward each agent container's journal to its hive collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent container writes a complete journal — 991 MB and nine days deep on this hive — that nothing outside it can read: the host-side per-container journal directory is an id-mapped bind mount, and journald writes nothing into it. So the reader has to run inside the container, and the path it would push to did not exist. Three tiers, one vertical slice, because any two of them alone are silent: - the agent container gains an `opentelemetry-collector` with a `journald` receiver aimed at its own journal and an exporter aimed at the same base address every in-process producer already exports to. - the hive collector gains `service.pipelines.logs`. Without it the `otlp` receiver answers 404 on `/v1/logs` — measured, and indistinguishable from a route that was never meant to exist. - the swarm collector gains a per-hive `logs/` pipeline beside `metrics/`. Without it the push is accepted, answered 200, and routed nowhere. The receiver's `directory` is stated rather than inherited, and that is the load-bearing line: its default is the RUNTIME journal (`/run/log/journal`), which in an agent container is empty. Left at the default this whole path validates, starts, reports healthy and forwards nothing. The assertion beside it covers the same silence from the other end — a `volatile` or `none` journald storage empties the directory the receiver reads. Attribution follows the tier that can prove it. The forwarder stamps `agent`, which no host-side reader could supply; `hive` is deliberately left to the swarm tier, which upserts it from whichever receiver accepted the sample, precisely so the label comes from something the sender cannot write. No `units` allowlist, unlike the swarm tier's journald receiver. That one needs one because the host's journal also holds an operator's own session; a container's journal is the harness and what the harness spawns. Measured volume is 20827 entries / 6.3 MB per agent per day, with nothing logging below `info` — so the receiver's `info` default filters nothing and there is no bill to justify a knob. Agent containers only, per the ruling on the issue: swarm services need one forwarder per service container and get re-measured once this works. Part of #3940. --- nix/agent-modules/otel.nix | 112 +++++++++++++++++++++++++++++++- nix/host-modules/otel.nix | 24 +++++++ nix/host-modules/swarm-otel.nix | 22 +++++++ 3 files changed, 156 insertions(+), 2 deletions(-) diff --git a/nix/agent-modules/otel.nix b/nix/agent-modules/otel.nix index 24867e6e..dbf62a38 100644 --- a/nix/agent-modules/otel.nix +++ b/nix/agent-modules/otel.nix @@ -1,6 +1,14 @@ # OpenTelemetry wiring for an agent container: the per-agent options the -# meta flake injects (host-driven from `services.hyperhive.otel.*`), and -# the environment every OTLP producer inside the container reads. +# meta flake injects (host-driven from `services.hyperhive.otel.*`), the +# environment every OTLP producer inside the container reads, and the +# collector that forwards this container's journal to the hive. +# +# Two directions, one endpoint. Metrics are PUSHED by processes in here +# that speak OTLP themselves; logs are not pushed by anyone, because +# nothing in a journal has an SDK. The forwarder is what turns the second +# into the first, and it has to run inside the container — a hive-level +# reader cannot see this journal at all (the host-side per-container +# directory is an id-mapped bind mount journald never writes into). # # Claude Code is one producer here, not the owner. `hive-metric` — and # any future in-container exporter — reads the same endpoint, protocol @@ -10,6 +18,7 @@ # options declared below. { lib, + pkgs, config, ... }: @@ -58,6 +67,11 @@ let // lib.optionalAttrs (cfg.metricIntervalMs != null) { OTEL_METRIC_EXPORT_INTERVAL = toString cfg.metricIntervalMs; }; + + # The exporter's NAME picks the wire protocol, so `protocol` has to be + # read here as well as handed to the SDKs above — same expression as the + # swarm tier's upstream exporter in nix/host-modules/swarm-otel.nix. + logExporter = if cfg.protocol == "grpc" then "otlp" else "otlphttp"; in { # OTEL stats export is configured ONCE at host level via @@ -164,5 +178,99 @@ in # while `NIX_REMOTE` — the same value, also set globally — is there. systemd.globalEnvironment = producerEnv; environment.variables = producerEnv; + + # The journal this container writes is reachable from inside it and + # from nowhere else, which is why the forwarder runs here rather than + # the hive reading the container's journal from outside: the host-side + # per-container directory is an id-mapped bind mount and journald + # writes nothing into it. + assertions = [ + { + # Sibling of the swarm collector's storage assertion in + # nix/host-modules/swarm-otel.nix, and it exists because the + # failure is silent at every layer: with a volatile journal the + # receiver below finds an empty directory, reads nothing, and the + # collector starts clean and stays healthy forever. + assertion = + !(lib.elem config.services.journald.storage [ + "volatile" + "none" + ]); + message = '' + hyperhive.otel.enable is on for agent ${userName}, but + services.journald.storage is + "${config.services.journald.storage}" in this container. + + The log forwarder reads /var/log/journal, which journald only + writes when it stores persistently: with "volatile" the journal + lives in /run/log/journal and with "none" there is none at all. + Either way the forwarder would ship nothing while looking + healthy. + ''; + } + ]; + + services.opentelemetry-collector = { + enable = true; + # Contrib, and not a preference: `journald` is a contrib receiver. + # The upstream default build has no way to read a journal at all. + package = pkgs.opentelemetry-collector-contrib; + # Runs `otelcol validate` at build time. ⚠️ A parser, not a wiring + # check — it accepts a pipeline naming a component the build lacks, + # and the collector then dies at startup. Same caveat as both other + # tiers; see nix/host-modules/otel.nix for the measurement. + validateConfigFile = true; + settings = { + receivers.journald = { + # ⚠️ STATED, and it must stay stated: the receiver's own default + # is the RUNTIME journal (`/run/log/journal`), which in an agent + # container is empty — journald stores persistently here, so + # every entry is under /var/log/journal. Dropping this line + # leaves a collector that validates, starts, reports healthy and + # forwards nothing. + directory = "/var/log/journal"; + # No `units` allowlist, unlike the swarm tier's receiver. That + # one needs one because the host's journal also holds an + # operator's own session; a container's journal is the harness + # and what the harness spawns, so there is no foreign traffic to + # filter out and an allowlist would only be a list to forget to + # update. + }; + + # Identity the hop above cannot supply. A host-side reader can say + # which machine a line came from; only a collector running as the + # agent can say which agent. `hive` and `swarm` are deliberately + # NOT stamped here even though the env above carries them for + # in-process producers: the swarm tier upserts `hive` from the + # receiver that accepted the sample, precisely so the label comes + # from something the sender cannot write. + processors.resource.attributes = [ + { + key = "service.name"; + value = "hyperhive-agent"; + action = "upsert"; + } + { + key = "agent"; + value = userName; + action = "upsert"; + } + ]; + + # The hive's collector, at the same base address every in-process + # producer already exports to. `endpoint` is a BASE the exporter + # appends `/v1/logs` to — the path an OTLP/HTTP receiver serves. + exporters.${logExporter} = { + endpoint = cfg.endpoint; + } + // lib.optionalAttrs (cfg.protocol == "http/json") { encoding = "json"; }; + + service.pipelines.logs = { + receivers = [ "journald" ]; + processors = [ "resource" ]; + exporters = [ logExporter ]; + }; + }; + }; }; } diff --git a/nix/host-modules/otel.nix b/nix/host-modules/otel.nix index 95ed69e1..65b90c3d 100644 --- a/nix/host-modules/otel.nix +++ b/nix/host-modules/otel.nix @@ -532,6 +532,30 @@ in ]; exporters = [ swarmName ]; }; + + # The second signal this tier carries, and the reason it exists + # is that an agent's journal is unreadable from here: the + # host-side per-container journal directory is an id-mapped bind + # mount journald never writes into, so each agent container runs + # its own forwarder (nix/agent-modules/otel.nix) and pushes over + # the `otlp` receiver above. Without this pipeline that receiver + # answers 404 on `/v1/logs` — measured, and indistinguishable + # from a route that was never meant to exist. + # + # One receiver, two pipelines: `otlp` accepts both signals and + # each pipeline takes the one it names. The exporter needs no + # change either — its `endpoint` is a base, and otlphttp appends + # `/v1/logs` to it exactly as it appends `/v1/metrics`. + service.pipelines.logs = { + receivers = [ "otlp" ]; + # `resourcedetection` and NOT `deltatocumulative`: that one is + # a metrics-only processor, and naming it here is a startup + # failure rather than a no-op. Host identity applies to both + # signals for the same reason — a line is only comparable + # across hives once it says which machine collected it. + processors = [ "resourcedetection" ]; + exporters = [ swarmName ]; + }; } // lib.optionalAttrs senderAuth { extensions.${authName} = { diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 090b158c..db813dcd 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -1255,6 +1255,28 @@ in exporters = exporterNames; }; } + # A hive's logs, and the receiver is the SAME one its metrics + # arrive on: a receiver is not per-signal, so `otlp/${h}` + # feeds this pipeline and `metrics/${h}` above with no second + # port, credential or audience. Which is also why the `hive` + # label stays trustworthy for logs without any new mechanism + # — it still comes from which receiver accepted the sample. + # + # The senders are agent containers, one forwarder each + # (nix/agent-modules/otel.nix), via their own hive's collector. + # Without this pipeline that whole path terminates here: the + # receiver accepts the push, answers 200, and the records go + # nowhere. + // lib.optionalAttrs collectLogs ( + lib.mapAttrs' ( + h: _: + lib.nameValuePair "logs/${h}" { + receivers = [ "otlp/${h}" ]; + processors = [ "resource/${h}" ]; + exporters = logExporterNames; + } + ) hivePorts + ) # Logs fan out exactly as metrics do: the swarm's store when it # runs, the operator's upstream when one is configured, both # when both. The local store is a destination rather than the From 07639fd364b4dea33873c7fc9a35d4804667b2ef Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 02:46:08 +0200 Subject: [PATCH 03/10] otel: evaluate the agent log forwarder in module-eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in this suite evaluated nix/agent-modules/ at all: every fixture was a host, so a typo in a rendered container config surfaced on a real deploy and nowhere else. This adds an `agent` constructor beside `hive`, off the same `nixosModules.agent-base` the meta flake hands a container. It also adds the suite's first two-hive fixture. Every existing one declares `swarm.hives.h1` alone, so a per-hive arm written against those passes on a hardcoded literal — which is exactly what the new per-hive logs pipeline needed covering. Eight cases, each paired with the control that makes it mean something: the absence arms with a presence half, the per-hive arm with a roster length check, because `lib.all` over an empty roster holds vacuously. Each was then shown to fail. Eight mutations across the three files — `directory` back to the runtime default, a pipeline naming no receiver, an exporter aimed at a loopback literal, an exporter name that stops reading `protocol`, a missing hive-tier logs pipeline, the metrics-only processor inside it, a per-hive pipeline hardcoded to one hive, and logs pipelines exporting to the metrics store — all caught, none survived, none skipped, each run's baseline green. Part of #3940. --- nix/module-eval.nix | 166 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 17e2e210..71931666 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -58,6 +58,32 @@ let ]; }).config; + # The other half of the tree. `nix/agent-modules/` is evaluated by nothing + # else in this suite — every fixture above is a host — so a rendered + # container config was only ever read by a real deploy. Same entry point + # the meta flake hands a container, so what this evaluates is what an + # agent gets. + # + # Note `hyperhive`, not `services.hyperhive`: an agent container's options + # live at the top level. + agent = + extra: + (nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.agent-base + { + fileSystems."/" = { + device = "/dev/null"; + fsType = "tmpfs"; + }; + boot.loader.grub.enable = false; + system.stateVersion = "25.11"; + hyperhive = lib.recursiveUpdate { user.name = "a1"; } extra; + } + ]; + }).config; + allLocal = hive { deploy.singleHostSwarm = true; }; bare = hive { }; withCi = hive { deploy.forgejo.ci.enable = true; }; @@ -373,6 +399,45 @@ let swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; }; + # The log path's three hops, one fixture each. Nothing carries a journal + # record end to end at eval time, so what these defend is the part no tier + # can check for itself: each hop's output is the next hop's input, and + # every mismatch between them is silent — a push accepted and routed + # nowhere, a receiver pointed at an empty directory, a pipeline that does + # not exist. + agentBridge = "http://10.42.0.1:4318"; + agentOtel = agent { + otel.enable = true; + otel.endpoint = agentBridge; + }; + # The same agent over the other wire protocol. An exporter's NAME is what + # selects it, so this is where a defined exporter and the pipeline's + # reference to it can drift apart. + agentOtelGrpc = agent { + otel.enable = true; + otel.endpoint = agentBridge; + otel.protocol = "grpc"; + }; + agentNoOtel = agent { }; + agentSettings = machine: machine.services.opentelemetry-collector.settings; + + # This hive's own collector, which is a HOST service — unlike the swarm + # tier's, which lives in a container and is read through `otelSettings`. + hiveOtel = hive { + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; + + # Two hives in the roster, which no other fixture here has: every one of + # them declares `swarm.hives.h1` alone, so a per-hive arm written against + # one of those passes on a hardcoded literal. + otelTwoHives = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + swarm.hives.h2.domain = "h2.t.local"; + }; + # A priority collision is a property of the *option*, not # of the merged value's interior — nix throws the moment the value is # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is @@ -1022,6 +1087,107 @@ let in named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; } + { + # The journald receiver's own default directory is the RUNTIME + # journal, and a container that stores persistently leaves that + # empty. At the default the forwarder validates, starts, reports + # healthy and ships nothing, so this one literal is the difference + # between the path working and silently not. + name = "the agent forwarder reads the persistent journal, not the runtime one"; + ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; + } + { + # The hop's two ends: what it reads, and where what it reads goes. + # The endpoint is compared against the value the fixture handed the + # option rather than a literal spelled here, so an exporter that + # stopped reading the option fails instead of matching a constant + # that travelled beside it. + name = "the agent forwarder ships the journal to the endpoint its hive gave it"; + ok = + let + s = agentSettings agentOtel; + p = s.service.pipelines.logs; + in + p.receivers == [ "journald" ] + && p.exporters != [ ] + && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; + } + { + # Presence control for the two cases above: with the switch off there + # is no collector in the container at all, so their passing is about + # the wiring rather than about a unit that renders regardless. + name = "an agent that has not opted into telemetry runs no collector"; + ok = !agentNoOtel.services.opentelemetry-collector.enable; + } + { + # `otlp` and `otlphttp` are different components and the protocol + # option picks which one is defined. A pipeline left naming the other + # is a startup failure; an exporter no pipeline names is silence. + name = "the agent forwarder's exporter and its pipeline agree on the protocol"; + ok = + let + s = agentSettings agentOtelGrpc; + in + (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; + } + { + # The tier in the middle. Its OTLP receiver takes both signals on one + # port, so without this pipeline an agent's push is answered 404 on + # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish- + # able from one with nothing to send. Compared against the metrics + # pipeline's exporters rather than a name, so the two signals cannot + # drift to different destinations. + name = "the hive collector forwards logs upstream, not only metrics"; + ok = + (hiveOtelPipelines ? logs) + && hiveOtelPipelines.metrics.exporters != [ ] + && hiveOtelPipelines.logs.receivers == [ "otlp" ] + && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters; + } + { + # `deltatocumulative` is metrics-only: naming it in a logs pipeline + # kills the collector at startup rather than doing nothing. The second + # clause is the control — the metrics pipeline still names it, so a + # pass means the two processor lists differ rather than that the + # processor left the module. + name = "the hive collector keeps the metrics-only processor out of its logs pipeline"; + ok = + !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) + && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; + } + { + # Read against the roster the fixture declares rather than against + # names spelled here: an arm naming `h1` passes on a single-hive + # config however the mapping is written. The length clause is what + # makes the `all` mean anything — over an empty roster it holds + # vacuously. + name = "the swarm collector routes every hive's logs, not just one"; + ok = + let + p = (otelSettings otelTwoHives).service.pipelines; + hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives; + in + lib.length hives == 2 + && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives; + } + { + # The same split as the metrics case above — defining an exporter and + # naming it are two lists — plus the half one shared list cannot have: + # the metrics store's exporter renders perfectly well inside a logs + # pipeline and posts journal records at an ingest route that is not + # for them. + name = "every logs pipeline sends to the log store and to no metrics one"; + ok = + let + s = otelSettings otelTwoHives; + logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines; + used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes)); + in + logPipes != { } + && builtins.elem "otlphttp/victorialogs" used + && !(builtins.elem "otlphttp/victoriametrics" used) + && lib.all (e: s.exporters ? ${e}) used; + } { # The store's seal is spread over six gates — the stanza, the # provisioning unit, two bind mounts, a device and an EnvironmentFile. From ccde8a2445d500e615089998c14f89b0cec3d70f Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 02:46:13 +0200 Subject: [PATCH 04/10] docs: the agent telemetry hop carries logs now, not only stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit observability.md scoped the whole feature to Claude Code statistics, so a reader asking what leaves an agent container would have concluded counters and nothing else. That page also documents the trust properties of the agent->collector hop, and those are what needs restating for a richer payload: same destination, same credential, same already-accepted ability to push arbitrary OTLP. What changes is how much detail leaves by default. docs/swarm/services.md needs no change. It already said each hive ships the journals of "the harness units inside every agent container" — which was not true until now. Part of #3940. --- docs/scheduler/observability.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/scheduler/observability.md b/docs/scheduler/observability.md index 0ce60ae5..9e63f669 100644 --- a/docs/scheduler/observability.md +++ b/docs/scheduler/observability.md @@ -1,8 +1,9 @@ # Observability (OpenTelemetry) -hyperhive has built-in support for exporting per-agent Claude Code statistics — -token usage, cost, tool call counts — to any OTLP-compatible collector via -Claude Code's built-in OpenTelemetry integration. +hyperhive has built-in support for exporting each agent container's telemetry to +any OTLP-compatible collector: Claude Code statistics — token usage, cost, tool +call counts — via Claude Code's built-in OpenTelemetry integration, and the +container's journal, forwarded by a collector running inside it. This is a **hive-wide** setting: one switch in the host NixOS config enables it for every agent container simultaneously. No per-agent opt-in or opt-out exists. @@ -67,6 +68,14 @@ that.** Treat it as a convenience for grouping dashboards, never as evidence of which container produced a sample: any agent that can reach this hive's collector can label its data as any other agent. +**Logs ride the same hop, and a journal carries more than a counter does.** Each +agent container forwards its own journal through this port — every unit in it at +`info` and above, not an allowlist. That's the harness, the MCP daemons and +whatever a tool call spawned, so command lines and error text now leave the +container where before only counts did. The trust boundary is unchanged (same +destination, same credential, and an agent could already send arbitrary OTLP); +what changes is how much detail leaves by default. + Worth spelling out, because two different hops are in play and only one of them is getting a credential: From 84203a3f9b88580e6231f05dc12baeab18e84c2f Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 03:18:44 +0200 Subject: [PATCH 05/10] hive-forge: let pr status take the PR number positionally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other PR-scoped verb takes the number as a positional — `pr show 42`, `pr comments 42`, `pr assign-reviewer 42 argus`. `pr status` alone required `--pr 42`, so whichever form you learn first is wrong for the other, and clap's error for the mistake suggests `-- --pr`, which would pass the literal string on as the next positional. The number is positional here too now. `--pr` stays, because it was the only spelling this verb had; `--sha` keeps its flag because the two are alternatives rather than one required argument. Two error strings in this file also named `pr-status`, a form that has refused to run since the subcommand rename. One of them is the message you get for passing neither argument — i.e. exactly when you are already unsure what the verb is called. The four new cases cover both halves: that clap accepts each spelling, and that something reads it. `target_pr` is named rather than inlined for exactly that reason — dropping the positional from the selection passed every parse-only case while leaving `pr status 42` reporting "pass a PR number". The conflict case earns its keep the same way: clap accepting an argument and clap ignoring it are indistinguishable from a passing parse, so `42 --sha abc` and `42 --pr 42` both have to be rejected. Closes #4182. --- hive-forge/src/verbs/pr_status.rs | 89 +++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/hive-forge/src/verbs/pr_status.rs b/hive-forge/src/verbs/pr_status.rs index 246517d3..3802f6cf 100644 --- a/hive-forge/src/verbs/pr_status.rs +++ b/hive-forge/src/verbs/pr_status.rs @@ -1,12 +1,12 @@ -//! `pr status --pr ` — one-stop PR health view: mergeable state, CI +//! `pr status ` — one-stop PR health view: mergeable state, CI //! checks, requested reviewers + review verdicts, and the last-comment //! timestamp. `--sha ` is a CI-only fast path for a raw commit. //! Removes the need for raw `curl` to the statuses / reviews endpoints, //! keeping forge access on the single `hive-forge` tool. //! -//! Exit code is a merge-readiness verdict for `--pr`: 0 only when CI is -//! green AND the PR is mergeable AND no review requests changes — so it -//! composes (`hive-forge pr status --pr 42 && …`). `--sha` mirrors the +//! Exit code is a merge-readiness verdict when given a PR: 0 only when +//! CI is green AND the PR is mergeable AND no review requests changes — +//! so it composes (`hive-forge pr status 42 && …`). `--sha` mirrors the //! CI verdict alone (0 = success). use anyhow::{Context, Result, bail}; @@ -21,26 +21,42 @@ use crate::verbs::{print_json, rfc3339}; pub struct Args { /// PR number — full health view (mergeable, CI, reviews, last /// comment). Mutually exclusive with `--sha`. + #[arg(value_name = "NUMBER", conflicts_with_all = ["pr", "sha"])] + number: Option, + /// Same as passing the number positionally. Kept because this verb + /// accepted only this spelling before the positional existed. #[arg(long, conflicts_with = "sha")] pr: Option, /// Explicit commit sha (or ref) — CI-only fast path. Mutually - /// exclusive with `--pr`. + /// exclusive with a PR number. #[arg(long)] sha: Option, } +impl Args { + /// The PR to inspect, whichever spelling asked for it. + /// + /// Named rather than inlined so the two spellings can be shown to + /// select the same PR: a test that only parses proves clap accepted + /// the positional, never that anything reads it. + fn target_pr(&self) -> Option { + self.number.or(self.pr) + } +} + /// # Errors /// -/// Returns an error if neither `--pr` nor `--sha` is given, or any forge -/// GET fails (PR lookup, combined status, reviews, comments). A +/// Returns an error if neither a PR number nor `--sha` is given, or any +/// forge GET fails (PR lookup, combined status, reviews, comments). A /// not-ready verdict is NOT an error — it's reported and reflected in /// the process exit code instead. pub fn run(client: &Client, args: Args) -> Result<()> { let repo = client.repo()?; - match (args.pr, args.sha) { + let target = args.target_pr(); + match (target, args.sha) { (Some(pr), _) => pr_status(client, repo, pr), (None, Some(sha)) => sha_status(client, &sha), - (None, None) => bail!("pr-status: pass one of --pr or --sha "), + (None, None) => bail!("pr status: pass a PR number (`pr status 42`) or --sha "), } } @@ -104,7 +120,7 @@ fn pr_status(client: &Client, repo: &str, pr: u64) -> Result<()> { .head .as_ref() .and_then(|h| h.sha.clone()) - .with_context(|| format!("pr-status: PR #{pr} has no head.sha"))?; + .with_context(|| format!("pr status: PR #{pr} has no head.sha"))?; let requested: Vec = pull .requested_reviewers @@ -389,6 +405,59 @@ fn combined_json(sha: &str, state: &str, statuses: &[Value]) -> Value { #[cfg(test)] mod tests { use super::*; + use clap::Parser; + + /// `Args` is a flattened arg group, not a `Parser`, so parsing it + /// standalone needs a wrapper. Without one these cases could only be + /// written against the whole binary's clap tree. + #[derive(Parser)] + struct Wrap { + #[command(flatten)] + args: Args, + } + + fn parse(argv: &[&str]) -> Result { + Wrap::try_parse_from(argv).map(|w| w.args) + } + + /// The property: the number is accepted the way every other + /// `pr ` takes it, and the older flag still works. + #[test] + fn the_pr_number_is_accepted_positionally_and_as_a_flag() { + assert_eq!(parse(&["x", "42"]).unwrap().number, Some(42)); + assert_eq!(parse(&["x", "--pr", "42"]).unwrap().pr, Some(42)); + } + + /// And something reads it. Parsing only proves clap accepted the + /// positional — dropping it from the selection leaves every other + /// case here green while `pr status 42` reports "pass a PR number". + #[test] + fn both_spellings_select_the_same_pr_and_sha_selects_none() { + assert_eq!(parse(&["x", "42"]).unwrap().target_pr(), Some(42)); + assert_eq!(parse(&["x", "--pr", "42"]).unwrap().target_pr(), Some(42)); + assert_eq!(parse(&["x", "--sha", "abc"]).unwrap().target_pr(), None); + } + + /// Control for the case above. Clap accepting an argument and clap + /// ignoring it look identical from a passing parse, so the conflicts + /// have to be shown to bite. + #[test] + fn a_pr_number_conflicts_with_sha_and_with_the_flag() { + assert!(parse(&["x", "42", "--sha", "abc"]).is_err()); + assert!(parse(&["x", "42", "--pr", "42"]).is_err()); + } + + /// Giving neither is a runtime error from `run`, not a parse error — + /// so `--sha` alone has to keep parsing. + #[test] + fn neither_form_parses_and_sha_alone_still_does() { + let empty = parse(&["x"]).unwrap(); + assert!(empty.number.is_none() && empty.pr.is_none() && empty.sha.is_none()); + assert_eq!( + parse(&["x", "--sha", "abc"]).unwrap().sha.as_deref(), + Some("abc") + ); + } #[test] fn short_sha_truncates() { From 23850e633bff66982e3950e4fcdee1f0750ad65c Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 03:35:18 +0200 Subject: [PATCH 06/10] docs: regenerate forge-cli.md for pr status's positional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docs/tools/forge-cli.md` is generated from the clap tree, so changing an argument or its doc comment changes that file too. The freshness check in `checks.hive-forge-docs` is what says so, and it is what failed here — after 8m44s, on a PR that was already approved. Regenerated, no hand edits. Two hunks, both the `pr status` section, and `nix fmt` leaves the result byte-identical so the formatter and the generator are not fighting over it. Part of #4182. --- docs/tools/forge-cli.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/tools/forge-cli.md b/docs/tools/forge-cli.md index 857817ef..5747f86a 100644 --- a/docs/tools/forge-cli.md +++ b/docs/tools/forge-cli.md @@ -548,12 +548,16 @@ Create a pull request PR health view: mergeable / CI / reviews -**Usage:** `hive-forge pr status [OPTIONS]` +**Usage:** `hive-forge pr status [OPTIONS] [NUMBER]` + +###### **Arguments:** + +* `` — PR number — full health view (mergeable, CI, reviews, last comment). Mutually exclusive with `--sha` ###### **Options:** -* `--pr ` — PR number — full health view (mergeable, CI, reviews, last comment). Mutually exclusive with `--sha` -* `--sha ` — Explicit commit sha (or ref) — CI-only fast path. Mutually exclusive with `--pr` +* `--pr ` — Same as passing the number positionally. Kept because this verb accepted only this spelling before the positional existed +* `--sha ` — Explicit commit sha (or ref) — CI-only fast path. Mutually exclusive with a PR number From e0d356e08a67cc685ca5bf0ae271f8fa8df3c5ae Mon Sep 17 00:00:00 2001 From: damocles Date: Fri, 11 Sep 2026 06:45:59 +0200 Subject: [PATCH 07/10] hive-forge-notify: fix notify.rs's dangling doc pointer the module doc pointed at docs/integrations/forge.md::Sources, a section that was never written. the actual rationale + host differences it promises already live in docs/integrations/github.md's Notifications section (auth scheme, cadence) -- point there instead of duplicating content across two docs. fixes #4189 --- hive-forge-notify/src/notify.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hive-forge-notify/src/notify.rs b/hive-forge-notify/src/notify.rs index 09ce6165..2e9bbee0 100644 --- a/hive-forge-notify/src/notify.rs +++ b/hive-forge-notify/src/notify.rs @@ -18,7 +18,7 @@ //! **Multi-source**: always the internal Forgejo, plus github.com when the //! agent has a PAT. Each source polls independently behind //! [`Source`]; everything below is shared. Rationale -//! + host differences: `docs/integrations/forge.md::Sources`. +//! + host differences: `docs/integrations/github.md::Notifications`. //! //! Activation gates, self-notification filtering, body excerpt + //! truncation + heading escape, wrapper formats (comment / review / From 4846c3b953d20b289d8e67d95de8575a96b20816 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 07:28:36 +0200 Subject: [PATCH 08/10] docs/gateway: name the loopback-only vhost class the TLS claims exclude MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TLS section read as universal — "the gateway always terminates TLS", "listens on httpsPort on every vhost". Measured against the rendered config of an all-local host: ten of eleven vhosts carry two listens and addSSL/forceSSL, and `grafana-metrics` carries one plain-HTTP listen on 127.0.0.1 with a single `= /metrics` location proxying grafana's unix socket for the local collector. The exception is right; a doc with no room for it is not. Someone asking "is anything here serving plain HTTP" was told no, when the useful answer is "no, except an internal class that is loopback-bound and deliberately exempt" — which tells them what to look for on the next host. One sentence where the modes are introduced rather than a patch to each sentence that inherits the assumption, plus "every vhost" -> "every routable vhost" where the self-signed default repeats it. Two neighbouring claims were checked at the same time and hold: /_matrix/ proxies to 127.0.0.1:8008, and .well-known/matrix/client returns the homeserver base_url with no port suffix on 443. Closes #4191 --- docs/networking/gateway.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index 1e225bde..4468b431 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -169,6 +169,13 @@ The gateway always terminates TLS — self-signed is the implicit floor when nothing else is configured, so there is no http-only mode. Three modes, selected by which (if any) external TLS source is set: +⚠️ **One vhost class is exempt, so expect it during a TLS audit.** A vhost bound to loopback for a +local consumer carries a single plain-HTTP listen and no TLS. `grafana-metrics` +(`nix/host-modules/swarm-grafana.nix`) is the only one in the tree today: it listens on `127.0.0.1` +alone and serves one `= /metrics` location from grafana's unix socket, for the collector on this +host to scrape. Nothing off-host can reach it, so TLS there protects nothing. Every vhost with a +routable name is covered by the modes below. + | mode | config | cert source | `.well-known` scheme | |---|---|---|---| | self-signed (default) | neither `tls.certDir` nor `tls.acme` set | host hive-CA signs a gateway leaf (RSA-4096) | `https` | @@ -202,7 +209,7 @@ Mutual exclusion: `tls.certDir` set together with `tls.acme.enable = true` fails ### Self-signed TLS (default) -On by default, and listens on `httpsPort` (default 443) on every vhost beside the plain-http `port` (default 80). +On by default, and listens on `httpsPort` (default 443) on every routable vhost beside the plain-http `port` (default 80). (The loopback-only exception is named under *TLS modes* above.) The issuer is a **host-held hive CA**, not a bare self-signed leaf. A host service (`hive-tls-ca.service`, from the `hive-tls` module) generates a long-lived CA (`services.hyperhive.deploy.hive-controller.tls.caValidityDays`, default ~20y) under `services.hyperhive.deploy.hive-controller.tls.stateDir` (default `/var/lib/hive-tls`), then signs a gateway **leaf** (`leafValidityDays`, default 30d) with it. `hive-gateway-self-signed-cert` then imports the leaf into nginx's state dir (`/var/lib/hive-gateway/tls/{cert,key}.pem`). From 9a94fa08e38c1b01f97364a03e7c834d9bd07bf3 Mon Sep 17 00:00:00 2001 From: damocles Date: Fri, 11 Sep 2026 08:22:12 +0200 Subject: [PATCH 09/10] hive-priv: fix stale 0750 in the tmpfiles.d /run/hyperhive entry sync_agent_tmpfiles wrote 'd /run/hyperhive 0750 hive-core hive-core -' into /etc/tmpfiles.d/hyperhive-agents.conf on every agent spawn/destroy -- five months after 30a2a2e9 moved the directory to 0751 so hive-admin members (no hive-core membership) can traverse into it to reach host.sock. the 0750 write itself always EROFSes (hive-priv's ReadWritePaths doesn't include /run/hyperhive, tracked separately as #4194 -- do not 'fix' that by adding it there, since a *succeeding* 0750 write would permanently break traversal on every host). so the directory only ever stayed at the correct 0751 by accident, preserved by a failing privileged call, and any mechanism that ever recreates the dir at the old baked-in RuntimeDirectoryMode default would un-mask this instantly. matching the tmpfiles line to 0751 removes the wrong write entirely: no mode to change, no fchmod, no EROFS. root cause found by atlas, digging in from the other end while verifying boundary.md's permission numbers against the running host. credit is entirely theirs -- I'd been chasing reboot-vs-restart theories on the issue thread without finding this. fixes #4078 --- hive-priv/src/main.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hive-priv/src/main.rs b/hive-priv/src/main.rs index 6060cffa..ae06640b 100644 --- a/hive-priv/src/main.rs +++ b/hive-priv/src/main.rs @@ -3019,7 +3019,17 @@ async fn sync_agent_tmpfiles(agents: &[AgentTmpfilesEntry]) -> Result<(String, S // Parent dirs — created with permissive mode so hive-c0re can make subdirs. // /run/hyperhive itself is also a RuntimeDirectory of hive-c0re.service; the // tmpfiles.d entry here ensures it exists before hive-c0re starts (boot race). - content.push_str("d /run/hyperhive 0750 hive-core hive-core -\n"); + // + // 0751, not 0750: must match hive-c0re.service's own `RuntimeDirectoryMode` + // and `docs/trust-boundary/boundary.md` — the extra `--x` on `other` is what + // lets a `hive-admin`-only user (no `hive-core` membership) traverse into the + // directory to reach `host.sock`; without it that user gets a permission + // denied opening the socket despite correct group membership on the socket + // itself. A mismatch here isn't just cosmetic: this line is regenerated and + // re-applied on every agent spawn/destroy via `systemd-tmpfiles --create`, + // so a stale `0750` here actively re-asserts the old, wrong mode far more + // often than a reboot does. + content.push_str("d /run/hyperhive 0751 hive-core hive-core -\n"); writeln!(content, "d {AGENT_RUNTIME_ROOT} 0755 hive-core hive-core -").ok(); // `hive-core`, not root: c0re does the `create_dir_all` for a new agent's // subdir itself, so a root-owned parent EACCESes on the first spawn of a From dd9e0bf0b07b8a0f29b5c7b83b429d5334145f49 Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 11 Sep 2026 06:04:18 +0200 Subject: [PATCH 10/10] module-eval: pin which switch enables each swarm-wide service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `deploy.allSwarmServices` derives nine service enables and appeared nowhere in the check, so the tier a service sits on was prose only. The tenth, the swarm controller, already had this exact pair of cases — it rides `singleHostSwarm` instead, and swarm-ui follows the controller. Four cases: the switch turns its nine on, a hive that does not host them runs none, an operator placing one elsewhere still wins over the `mkDefault`, and hosting the shared services does not make a hive the swarm's control plane. The roster is counted before it is read: `lib.all` over an empty set holds vacuously, so a roster that lost a member would otherwise turn the case green by measuring nothing. Closes #4186 --- nix/module-eval.nix | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 71931666..91130618 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -453,8 +453,85 @@ let in builtins.foldl' (acc: v: builtins.seq v acc) true vals; + # The swarm's shared services hosted HERE without the all-local mode — a + # services box with hives elsewhere, the shape ./host-modules/ + # swarm-required-services.nix documents the switch for. + swarmServicesHere = hive { deploy.allSwarmServices = true; }; + + # Same, with one of those services placed on another host. Every derivation + # in that module is `mkDefault` so this stays expressible. + swarmServicesBaoElsewhere = hive { + deploy.allSwarmServices = true; + deploy.bao.enable = false; + }; + + # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own + # option and is NOT under `deploy` — spelled at the wrong path it would be + # undeclared rather than false, and a roster that quietly loses a member is + # what the count guard in the cases below exists to catch. Its membership + # here is deliberate and was the fix for a gap, not an oversight: the hive + # tier lands wherever the swarm services do. + swarmServiceEnables = + machine: + let + h = machine.services.hyperhive; + in + { + matrix = h.deploy.matrix.enable; + otel = h.otel.enable; + authelia = h.deploy.authelia.enable; + nats = h.deploy.nats.enable; + swarm-otel = h.deploy.swarm-otel.enable; + victoriametrics = h.deploy.victoriametrics.enable; + grafana = h.deploy.grafana.enable; + victorialogs = h.deploy.victorialogs.enable; + bao = h.deploy.bao.enable; + }; + # Each case: a name stating the property, and `ok`. cases = [ + { + # `lib.all` over an empty set holds vacuously, so the roster is counted + # before it is read: a helper that lost a member would otherwise turn + # this case green by measuring nothing. + name = "hosting the swarm's shared services turns on every service that switch owns"; + ok = + let + es = swarmServiceEnables swarmServicesHere; + in + lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es); + } + { + name = "a hive that does not host the swarm's shared services runs none of them"; + ok = + let + es = swarmServiceEnables bare; + in + lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es); + } + { + # The switch fills in for an operator who has not spoken and yields to + # one who has — that is what keeps a shared service placeable on a host + # of its own. A plain assignment or `mkForce` would satisfy both cases + # above and break this one. `nats` is the control: without it the case + # also passes on a fixture where nothing came on at all. + name = "placing one shared service elsewhere survives the switch that would enable it"; + ok = + let + es = swarmServiceEnables swarmServicesBaoElsewhere; + in + !es.bao && es.nats; + } + { + # The controller sits on the OTHER tier: `singleHostSwarm` places it + # (./host-modules/local-defaults.nix) and swarm-ui follows the + # controller. Pinned so that moving a service between tiers is a + # decision someone makes rather than a merge nobody reads. + name = "hosting the swarm's shared services does not make a hive the swarm's control plane"; + ok = + !swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable + && !swarmServicesHere.services.hyperhive.deploy.swarm-ui.enable; + } { name = "a hive that has not opted into all-local runs no swarm controller"; ok = !bare.services.hyperhive.deploy.swarm-controller.enable;