From c2c61b614628f53c5b5785ab96a54885fc3b6deb Mon Sep 17 00:00:00 2001 From: atlas Date: Fri, 9 Oct 2026 23:12:28 +0200 Subject: [PATCH] hive status publishing: read the hive's OIDC secret from the store, retry until it is there hive-c0re publishes this hive's status to the swarm queue as the OIDC client `hive-`. Its secret came from `deploy.hive-controller.statusPublish.clientSecretFile`, which defaulted to authelia's minted file only on a host running both authelia and the queue. Every other hive had no secret and never published, so the controller showed it `never_reported`. Now `swarm-secret-publish`, beside authelia, also copies each hive's `hive-` secret to `swarm/hives//oidc/client`. Its policy already allows `swarm/hives/*`. hive-c0re reads that path under the hive's own store leaf, which the existing `swarm/hives//*` grant covers, and holds the value in memory. The shared queue client no longer remembers a failed first connect for the life of the process. A failed attempt is retried with a backoff from 5s, doubling to a cap of 5 minutes; the dashboard banner stays up until an attempt succeeds. The two status publishers wait for the client instead of giving up on the first `None`, so a secret published after boot is picked up without a restart. Removed: the `statusPublish.clientSecretFile` option (both spellings, now removed options with a message), its same-host default, its LoadCredential, the assertion tying it to the other two coordinates, and `swarm_queue_client::QueueConfig::from_env`, whose only caller was this read. Publishing is now on wherever the queue URL and token endpoint are set and the hive holds a store leaf. Refs #5015 --- docs/swarm/README.md | 33 +- docs/swarm/credentials.md | 1 + docs/swarm/secrets.md | 1 + hive-c0re/src/lifecycle/host_config.rs | 3 +- hive-c0re/src/swarm_agent_status.rs | 6 +- hive-c0re/src/swarm_queue.rs | 429 ++++++++++++++++-- hive-c0re/src/swarm_status.rs | 7 +- nix/host-modules/deploy.nix | 35 +- .../glue-queue-agent-credential.nix | 13 +- nix/host-modules/hive-c0re/default.nix | 24 +- nix/host-modules/hive-c0re/environment.nix | 19 +- nix/host-modules/swarm-secret-publisher.nix | 35 +- nix/host-modules/swarm.nix | 128 +----- nix/module-eval/nats-tls.nix | 86 ++-- nix/module-eval/secret-publisher.nix | 18 + swarm-logs/src/auth.rs | 17 +- swarm-queue-client/README.md | 27 +- swarm-queue-client/src/lib.rs | 105 +---- swarm-secret-client/src/queue.rs | 58 +++ 19 files changed, 647 insertions(+), 398 deletions(-) diff --git a/docs/swarm/README.md b/docs/swarm/README.md index bca5ebe8..23c9b8b0 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -88,25 +88,26 @@ nothing has to re-publish. ### Making a hive report -Three options on the **hive**. They sit in two namespaces, because two of them -are facts about _this machine_ and one is the swarm's single address: +Two options on the **hive**, plus its store identity. They sit in two +namespaces, because the queue's URL is a fact about _this machine_ and the +token endpoint is the swarm's single address: -| option | what to set it to | -| ------------------------------------------------------- | --------------------------------------------- | -| `deploy.hive-controller.statusPublish.natsUrl` | `tls://:` | -| `swarm.statusPublish.tokenEndpoint` | the swarm IdP's `/api/oidc/token` | -| `deploy.hive-controller.statusPublish.clientSecretFile` | path to this hive's client secret, plaintext | +| option | what to set it to | +| ---------------------------------------------- | --------------------------------------------- | +| `deploy.hive-controller.statusPublish.natsUrl` | `tls://:` | +| `swarm.statusPublish.tokenEndpoint` | the swarm IdP's `/api/oidc/token` | -The queue URL and the token endpoint default to the swarm's own addresses on -every hive, so there is nothing to set for them. The secret is what turns -publishing on: a hive without it doesn't publish. A secret without the other -two is an eval error rather than a hive that quietly never reports. +Both default to the swarm's own addresses on every hive, so there is nothing to +set for them. The client secret isn't an option: hive-c0re reads it from the +swarm's secret store, at `swarm/hives//oidc/client`, under the hive's +own store leaf (`deploy.bao.clientCertFile`). `swarm-secret-publish`, on the +host that runs authelia, puts it there. A hive without a store leaf doesn't +publish; setting `natsUrl` to null turns publishing off too. -On a host that runs the queue and the IdP, the secret defaults to the local one. -Any other hive needs the secret to physically be there, because the swarm -doesn't distribute it. Copy `hive-.secret` out of the swarm host's -`deploy.authelia.hostClientSecretDir` with whatever secret management the -deployment already uses. +Until the secret is in the store, hive-c0re shows a `swarm_queue_config` +warning and tries again, waiting 5 seconds after the first failure and doubling +up to 5 minutes. It starts publishing as soon as a read succeeds, with no +restart. The queue URL is the same string on every hive. The queue accepts TLS only, with a certificate for `swarm.nats.domain` (default `nats.`) and diff --git a/docs/swarm/credentials.md b/docs/swarm/credentials.md index 93e655d6..be0af1a0 100644 --- a/docs/swarm/credentials.md +++ b/docs/swarm/credentials.md @@ -70,6 +70,7 @@ of the cell says how. | `swarm/hives//matrix/sender-token` | `swarm-controller`, with the swarm's appservice token, for every hive in its directory in a five-minute pass | `swarm-controller` under its own certificate, before it decides whether to mint, and hive-c0re's `stored_sender_token()`, under the hive's own certificate | ✅ the controller's pass re-mints when the stored token is missing, unknown to the homeserver, or someone else's | ✅ hive-c0re's matrix sweep reads the store every run and overwrites its token file when the store's token differs | | `swarm/hives//forge/token` | `swarm-controller`, with the forge's admin API, for every hive in its directory in a five-minute pass | `swarm-controller` under its own certificate, before it decides whether to mint, and hive-c0re's `forge::reader_token()`, under the hive's own certificate | ✅ the controller's pass re-mints when the stored token is missing or no longer matches the forge (last eight characters and scopes) | ✅ hive-c0re reads the store on every forge read | | `swarm/hives//queue/agent` | authelia | `swarm-bao-queue-agent` on the hive's host, under its own per-hive certificate; no agent's policy reaches it | must be stated | must be stated | +| `swarm/hives//oidc/client` | authelia, at its first boot; `swarm-secret-publish` copies it in | hive-c0re, under the hive's own certificate, when it first connects to the queue | ❌ authelia mints it once. A re-mint is republished by `swarm-secret-publish`'s path unit | ❌ read until the first connect succeeds, retried with a backoff of up to five minutes while it's missing; then held until hive-c0re restarts | | `swarm/services//oidc/client` | authelia | that service's own reader, under its own certificate — the queue responder's under `swarm-nats-auth`'s, the homeserver's under `swarm-matrix-ctl`'s; the bao UI's by the store's granter | must be stated | must be stated | | `swarm/services/ci-runner/registration-token` | `swarm-controller`, from the forge's admin API, at start and in a five-minute pass; its grant on the path is write-only | `swarm-bao-ci-runner` on the runner's host, under the runner's own certificate, into a file only `swarm-ci-runner-register` in the runner's container loads; no other principal's policy reads it | ❌ the forge hands out one token until an admin resets it; the controller writes a reset token within five minutes | ❌ `swarm-bao-ci-runner` reads it when it starts, at boot or deploy, and restarts the runner when the value changed | | _(not in the store)_ a hive's mTLS leaf | the store's own PKI, or an operator placing it by hand | its own client, off disk — the exception above, because it's what makes every other row's pull possible | must be stated | must be stated | diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index a7502fa3..f1114b99 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -52,6 +52,7 @@ neither is a renaming of the other. | the swarm collector's copy of its OIDC secret | `swarm-bao-otel-oidc.service` reads it out of the swarm secret store, **on every host that runs the collector and holds a store identity** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | same unit, same path — one route, co-located or not. A collector with no store identity has `clientSecretFile == null`, its already-supported unauthenticated-push degrade — see below | | Grafana's copy of its OIDC secret | `swarm-bao-grafana-oidc.service` reads it out of the swarm secret store, **on every host that runs Grafana** | `/var/lib/grafana-oidc/.secret` inside the `swarm-grafana` container | same unit, same path — one route, co-located or not. Nothing for an operator to place beyond this host's store leaf, see below | | the swarm controller's copy of its OIDC secret | `swarm-controller` reads it out of the swarm secret store at start, under its own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the controller's store leaf (`deploy.swarm-controller.baoClientCertFile`) | +| a hive's copy of its own OIDC secret | hive-c0re reads it out of the swarm secret store when it first connects to the queue, under the hive's own leaf | the daemon's memory only; never on disk | same read, co-located or not. Nothing for an operator to place beyond the hive's store leaf (`deploy.bao.clientCertFile`) | | the swarm controller's forge token | the forge's `forgejo-swarm-controller-account`, in-container, once | `/var/lib/forgejo/swarm-controller-token` inside the forge container, the minter's record that it has minted; `swarm-bao-forge-publisher` copies it into the swarm secret store at `swarm/controller/swarm-controller/forge/token` each time the forge container starts and each time the mint writes the file; the controller holds it in memory only | nothing to place beyond the controller's store leaf. A forge host without the store needs the publisher's leaf placed (`deploy.bao.forgePublisherClientCertFile`) | | the queue responder's copy of its OIDC secret | the responder reads it out of the swarm secret store itself as it starts, under its own leaf, and again when authelia refuses it | the daemon's memory only; never on disk | same read, wherever authelia runs. Nothing for an operator to place beyond the responder's store leaf (`deploy.nats.authBaoClientCertFile`) | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | diff --git a/hive-c0re/src/lifecycle/host_config.rs b/hive-c0re/src/lifecycle/host_config.rs index f8a10349..236413fa 100644 --- a/hive-c0re/src/lifecycle/host_config.rs +++ b/hive-c0re/src/lifecycle/host_config.rs @@ -272,8 +272,7 @@ fn queue_agent_credentials(agent_name: &str, dir: Option<&Path>) -> Vec) { }; tokio::spawn(async move { - let Some(client) = crate::swarm_queue::client().await else { + let Some(client) = crate::swarm_queue::wait_for_client().await else { return; }; diff --git a/hive-c0re/src/swarm_queue.rs b/hive-c0re/src/swarm_queue.rs index a79a069e..664ca87b 100644 --- a/hive-c0re/src/swarm_queue.rs +++ b/hive-c0re/src/swarm_queue.rs @@ -10,63 +10,400 @@ //! crate in the first place, one layer up. This module is that same move //! made again, this time between two consumers *inside* one process. //! +//! The queue's address, the token endpoint and the client id come from the +//! environment. The client secret comes from the swarm's secret store, at +//! `swarm_secret_client::queue::hive_client_path`, read under this hive's own +//! certificate and held only in memory. `swarm-secret-publish`, on the host +//! that runs authelia, puts it there. +//! //! Connects lazily on first use rather than at boot — nothing here //! blocks `hive-c0re` starting up on hosts with no queue configured, -//! which is the ordinary case. +//! which is the ordinary case. A failed attempt is retried after a +//! backoff capped at [`MAX_RETRY_DELAY`], so a secret published after this +//! daemon started is picked up without a restart. -use tokio::sync::OnceCell; +use std::future::Future; +use std::path::PathBuf; +use std::time::Duration; -/// Env var prefix for this daemon's swarm-queue credentials — see -/// [`swarm_queue_client::QueueConfig::from_env`]. All four or none. +use anyhow::{Context, Result, anyhow, bail}; +use swarm_queue_client::{ClientSecret, QueueConfig}; +use swarm_secret_client::queue::{self, HiveClientCredential}; +use tokio::sync::Mutex; +use tokio::time::Instant; + +/// Env var prefix for this daemon's swarm-queue coordinates — see +/// [`coordinates_from`]. const ENV_PREFIX: &str = "HIVE_C0RE"; -static CLIENT: OnceCell> = OnceCell::const_new(); +/// Wait after the first failed attempt; doubled after each further one. +const FIRST_RETRY_DELAY: Duration = Duration::from_secs(5); -/// The shared swarm-queue client, connecting on first call and memoized -/// for the rest of the process's life. +/// The longest wait between two attempts. +const MAX_RETRY_DELAY: Duration = Duration::from_mins(5); + +/// The banner kind a failed attempt raises, cleared by the next success. +const BANNER: &str = "swarm_queue_config"; + +static CLIENT: Retrying = Retrying::new(FIRST_RETRY_DELAY, MAX_RETRY_DELAY); + +/// The shared swarm-queue client, or `None` when there is none right now. /// -/// `None` covers both "no queue configured" (the ordinary case, logged -/// once at `info`) and "config present but connecting failed" (bannered -/// once via [`crate::warnings::set_boot_warning`] the first time this is -/// called) — either way, a caller with `None` should just skip whatever -/// it was about to publish. No caller needs to distinguish the two: both -/// mean "this hive is not offering anything to the swarm right now." +/// `None` covers "no queue configured" (the ordinary case, logged once at +/// `info`), a deployment bug in the coordinates (bannered, and permanent for +/// this process), and an attempt that failed or is still backing off. A +/// caller with `None` should just skip whatever it was about to publish. +/// Never waits out a backoff: see [`wait_for_client`] for that. pub async fn client() -> Option { - CLIENT.get_or_init(connect_once).await.clone() + match CLIENT.get(connect_once).await { + Outcome::Ready(client) => Some(client), + Outcome::Off | Outcome::NotBefore(_) => None, + } } -async fn connect_once() -> Option { - let cfg = match swarm_queue_client::QueueConfig::from_env(ENV_PREFIX) { - Ok(Some(cfg)) => cfg, - Ok(None) => { - tracing::info!("no swarm queue configured; this hive offers nothing upward"); - return None; - } - Err(e) => { - // A one-shot startup step with no later retry to clear it — - // exactly what `set_boot_warning` is for. `chain`, not - // `{:#}`: this is `swarm_queue_client::Error`, whose - // `Display` ignores the alternate flag (see `chain`'s own - // doc comment), so `{:#}` would drop which env vars are - // actually missing. - crate::warnings::set_boot_warning( - "swarm_queue_config", - "warn", - format!("swarm queue is off: {}", swarm_queue_client::chain(&e)), - ); - return None; - } - }; - - match swarm_queue_client::connect(cfg).await { - Ok(client) => Some(client), - Err(e) => { - crate::warnings::set_boot_warning( - "swarm_queue_config", - "warn", - format!("swarm queue is off: {}", swarm_queue_client::chain(&e)), - ); - None +/// The shared swarm-queue client, waiting through every backoff until there +/// is one. `None` only when this hive publishes nothing to a queue at all. +pub async fn wait_for_client() -> Option { + loop { + match CLIENT.get(connect_once).await { + Outcome::Ready(client) => return Some(client), + Outcome::Off => return None, + Outcome::NotBefore(at) => tokio::time::sleep_until(at).await, } } } + +/// What one connection attempt came to. +enum Attempt { + /// Nothing to connect to, for the life of the process. + Off, + Ready(T), + /// Worth trying again later. + Failed(anyhow::Error), +} + +/// What [`Retrying::get`] answers. +enum Outcome { + Off, + Ready(T), + /// The last attempt failed; the next one is not tried before this. + NotBefore(Instant), +} + +enum Slot { + Off, + Ready(T), + /// `failures` consecutive attempts have failed. `banner` holds the + /// dashboard warning until the slot leaves this state. + Failing { + failures: u32, + not_before: Instant, + banner: Option, + }, +} + +/// A value made by an attempt that can fail: a success or an `Off` is kept +/// for the life of the process, a failure only until its backoff runs out. +struct Retrying { + slot: Mutex>>, + first_delay: Duration, + max_delay: Duration, +} + +impl Retrying { + const fn new(first_delay: Duration, max_delay: Duration) -> Self { + Self { + slot: Mutex::const_new(None), + first_delay, + max_delay, + } + } + + /// The kept value, or one new `attempt` if the last one's backoff has run + /// out. Concurrent callers wait on one attempt rather than each making + /// their own. + async fn get(&self, attempt: F) -> Outcome + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let mut slot = self.slot.lock().await; + let failures = match &*slot { + None => 0, + Some(Slot::Off) => return Outcome::Off, + Some(Slot::Ready(value)) => return Outcome::Ready(value.clone()), + Some(Slot::Failing { + failures, + not_before, + .. + }) => { + if Instant::now() < *not_before { + return Outcome::NotBefore(*not_before); + } + *failures + } + }; + match attempt().await { + Attempt::Off => { + *slot = Some(Slot::Off); + Outcome::Off + } + Attempt::Ready(value) => { + if failures > 0 { + tracing::info!(failures, "swarm queue: connected after earlier failures"); + } + *slot = Some(Slot::Ready(value.clone())); + Outcome::Ready(value) + } + Attempt::Failed(e) => { + let failures = failures.saturating_add(1); + let delay = self.delay_after(failures); + let not_before = Instant::now() + delay; + let message = format!("swarm queue is off, retrying in {delay:?}: {e:#}"); + tracing::warn!(failures, "{message}"); + // Updating the held guard keeps the banner up without a + // moment where it is cleared and re-raised. + let banner = match slot.take() { + Some(Slot::Failing { + banner: Some(banner), + .. + }) => { + banner.update("warn", message); + banner + } + _ => crate::warnings::set_warning(BANNER, "warn", message), + }; + *slot = Some(Slot::Failing { + failures, + not_before, + banner: Some(banner), + }); + Outcome::NotBefore(not_before) + } + } + } + + /// `first_delay` doubled per failure after the first, capped at + /// `max_delay`. + fn delay_after(&self, failures: u32) -> Duration { + let doublings = failures.saturating_sub(1).min(16); + self.first_delay + .saturating_mul(1 << doublings) + .min(self.max_delay) + } +} + +async fn connect_once() -> Attempt { + let coordinates = match coordinates_from(|name| std::env::var(name).ok()) { + Ok(Some(c)) => c, + Ok(None) => { + tracing::info!("no swarm queue configured; this hive offers nothing upward"); + return Attempt::Off; + } + Err(e) => { + // A deployment bug, which no retry clears. + crate::warnings::set_boot_warning(BANNER, "warn", format!("swarm queue is off: {e:#}")); + return Attempt::Off; + } + }; + let secret = match fetch_secret().await { + Ok(secret) => secret, + Err(e) => return Attempt::Failed(e), + }; + match swarm_queue_client::connect(coordinates.with_secret(secret)).await { + Ok(client) => Attempt::Ready(client), + // `chain`, not `{:#}`: this is `swarm_queue_client::Error`, whose + // `Display` ignores the alternate flag. + Err(e) => Attempt::Failed(anyhow!(swarm_queue_client::chain(&e))), + } +} + +/// Everything in a [`QueueConfig`] except the secret. +#[derive(Debug, PartialEq, Eq)] +struct Coordinates { + url: String, + token_endpoint: String, + client_id: String, + ca_file: Option, +} + +impl Coordinates { + fn with_secret(self, secret: String) -> QueueConfig { + QueueConfig { + url: self.url, + token_endpoint: self.token_endpoint, + client_id: self.client_id, + client_secret: ClientSecret::Value(secret), + ca_file: self.ca_file, + } + } +} + +/// `_NATS_URL`, `_OIDC_TOKEN_ENDPOINT` and `_OIDC_CLIENT_ID`, all +/// or none; `_OIDC_CA_FILE` optionally. +fn coordinates_from(var: impl Fn(&str) -> Option) -> Result> { + let url = var(&format!("{ENV_PREFIX}_NATS_URL")); + let token_endpoint = var(&format!("{ENV_PREFIX}_OIDC_TOKEN_ENDPOINT")); + let client_id = var(&format!("{ENV_PREFIX}_OIDC_CLIENT_ID")); + let ca_file = var(&format!("{ENV_PREFIX}_OIDC_CA_FILE")).map(PathBuf::from); + match (url, token_endpoint, client_id) { + (None, None, None) => Ok(None), + (Some(url), Some(token_endpoint), Some(client_id)) => Ok(Some(Coordinates { + url, + token_endpoint, + client_id, + ca_file, + })), + _ => bail!( + "swarm queue is half-configured: {ENV_PREFIX}_NATS_URL, \ + {ENV_PREFIX}_OIDC_TOKEN_ENDPOINT and {ENV_PREFIX}_OIDC_CLIENT_ID must be set \ + together or not at all" + ), + } +} + +/// One login and one read, under this hive's own certificate. An empty value +/// is refused like an absent one: it would reach authelia as the secret and +/// come back as `invalid_client`, which names neither the store nor the +/// publisher. +async fn fetch_secret() -> Result { + let Some(hive) = crate::container_view::hive_swarm_names().0 else { + bail!("HYPERHIVE_HIVE_NAME is unset, so there is no store path for this hive's secret"); + }; + let path = queue::hive_client_path(&hive)?; + let store = swarm_secret_client::SecretStore::from_env(&hive) + .await + .context("logging in to the swarm secret store")?; + let stored: Option = store + .read_optional(&path) + .await + .with_context(|| format!("reading {path}"))?; + match stored { + Some(c) if !c.value.trim().is_empty() => Ok(c.value.trim().to_owned()), + _ => bail!( + "nothing at {path}: swarm-secret-publish, on the host that runs authelia, has not \ + published this hive's client secret" + ), + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let map: HashMap = pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(); + move |name| map.get(name).cloned() + } + + const FULL: [(&str, &str); 3] = [ + ("HIVE_C0RE_NATS_URL", "tls://nats.example:4222"), + ( + "HIVE_C0RE_OIDC_TOKEN_ENDPOINT", + "https://auth.example/api/oidc/token", + ), + ("HIVE_C0RE_OIDC_CLIENT_ID", "hive-alpha"), + ]; + + #[test] + fn no_coordinates_is_no_queue() { + assert_eq!( + coordinates_from(env(&[])).expect("absent is not an error"), + None + ); + } + + #[test] + fn a_half_set_environment_is_an_error_naming_the_variables() { + let err = coordinates_from(env(&FULL[..2])).expect_err("a partial set is not absent"); + assert!( + format!("{err}").contains("HIVE_C0RE_OIDC_CLIENT_ID"), + "{err}" + ); + } + + #[test] + fn a_full_environment_carries_the_stored_secret_as_a_value() { + let cfg = coordinates_from(env(&FULL)) + .expect("complete") + .expect("configured") + .with_secret("s3cret".to_owned()); + assert_eq!(cfg.client_id, "hive-alpha"); + assert!(matches!(&cfg.client_secret, ClientSecret::Value(v) if v == "s3cret")); + } + + /// The secret is not in the store when this daemon first asks, and is + /// later: the later value is used, with no restart in between. + #[tokio::test] + async fn a_failed_first_read_is_retried_and_its_later_secret_used() { + let shared = Retrying::::new(Duration::ZERO, Duration::ZERO); + + let first = shared + .get(|| async { Attempt::Failed(anyhow!("nothing at the path yet")) }) + .await; + assert!( + matches!(first, Outcome::NotBefore(_)), + "the first read failed" + ); + + let second = shared + .get(|| async { Attempt::Ready("published-later".to_owned()) }) + .await; + assert!( + matches!(&second, Outcome::Ready(s) if s == "published-later"), + "the read after the failure is made and its secret used" + ); + + let kept = shared + .get(|| async { panic!("a success is kept, not re-read") }) + .await; + assert!(matches!(&kept, Outcome::Ready(s) if s == "published-later")); + } + + #[tokio::test(start_paused = true)] + async fn a_failure_is_not_retried_before_its_backoff_runs_out() { + let shared = Retrying::::new(Duration::from_secs(5), Duration::from_mins(5)); + let failed = shared + .get(|| async { Attempt::Failed(anyhow!("down")) }) + .await; + let Outcome::NotBefore(at) = failed else { + panic!("a failure backs off"); + }; + assert_eq!(at - Instant::now(), Duration::from_secs(5)); + + let early = shared + .get(|| async { panic!("no attempt inside the backoff") }) + .await; + assert!(matches!(early, Outcome::NotBefore(t) if t == at)); + + tokio::time::advance(Duration::from_secs(5)).await; + let due = shared + .get(|| async { Attempt::Ready("up".to_owned()) }) + .await; + assert!(matches!(&due, Outcome::Ready(s) if s == "up")); + } + + #[test] + fn the_backoff_doubles_up_to_its_cap() { + let shared = Retrying::::new(Duration::from_secs(5), Duration::from_mins(5)); + let delays: Vec = (1..=8).map(|n| shared.delay_after(n).as_secs()).collect(); + assert_eq!(delays, [5, 10, 20, 40, 80, 160, 300, 300]); + assert_eq!(shared.delay_after(u32::MAX), Duration::from_mins(5)); + } + + #[tokio::test] + async fn off_is_kept_for_the_life_of_the_process() { + let shared = Retrying::::new(Duration::ZERO, Duration::ZERO); + assert!(matches!( + shared.get(|| async { Attempt::Off }).await, + Outcome::Off + )); + let again = shared.get(|| async { panic!("off is not re-tried") }).await; + assert!(matches!(again, Outcome::Off)); + } +} diff --git a/hive-c0re/src/swarm_status.rs b/hive-c0re/src/swarm_status.rs index 67f7ac50..13a1d455 100644 --- a/hive-c0re/src/swarm_status.rs +++ b/hive-c0re/src/swarm_status.rs @@ -75,10 +75,9 @@ pub fn spawn( }; tokio::spawn(async move { - let Some(client) = crate::swarm_queue::client().await else { - // Absent or failed — either way already handled (an `info` - // log or a `swarm_queue_config` banner) by the shared - // connector; nothing left to report here. + let Some(client) = crate::swarm_queue::wait_for_client().await else { + // No queue configured, already logged or bannered by the + // shared connector; nothing left to report here. return; }; diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 7f819b20..b75f9e09 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -32,6 +32,14 @@ let deployCfg = config.services.hyperhive.deploy; + # Shared by the two removed spellings of the status-publishing secret below. + statusSecretRemoved = '' + hive-c0re reads this hive's status-publishing client secret from the + swarm's secret store, under the hive's own certificate + (deploy.bao.clientCertFile), and swarm-secret-publish on the host that runs + authelia puts it there. Remove this definition; the file it named is read + by nothing now and can be deleted. + ''; in { imports = [ @@ -115,19 +123,28 @@ in [ "services" "hyperhive" "swarm" "nats" "enable" ] [ "services" "hyperhive" "deploy" "nats" "enable" ] ) - # Two of statusPublish's three coordinates. `tokenEndpoint` stays under - # `swarm.*`: it is the swarm IdP's one address, identical on every hive, - # where these two are "the queue as seen from here" and "the secret on - # this disk". The all-or-nothing assertion in ./swarm.nix spans the split - # deliberately and names all three paths in its message. + # `tokenEndpoint` stays under `swarm.*`: it is the swarm IdP's one + # address, identical on every hive, where this is "the queue as seen from + # here". (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "statusPublish" "natsUrl" ] [ "services" "hyperhive" "deploy" "hive-controller" "statusPublish" "natsUrl" ] ) - (lib.mkRenamedOptionModule - [ "services" "hyperhive" "swarm" "statusPublish" "clientSecretFile" ] - [ "services" "hyperhive" "deploy" "hive-controller" "statusPublish" "clientSecretFile" ] - ) + (lib.mkRemovedOptionModule [ + "services" + "hyperhive" + "swarm" + "statusPublish" + "clientSecretFile" + ] statusSecretRemoved) + (lib.mkRemovedOptionModule [ + "services" + "hyperhive" + "deploy" + "hive-controller" + "statusPublish" + "clientSecretFile" + ] statusSecretRemoved) # Both old spellings of "run a hive here" forward to the one switch. (lib.mkRenamedOptionModule [ "services" "hyperhive" "enable" ] diff --git a/nix/host-modules/glue-queue-agent-credential.nix b/nix/host-modules/glue-queue-agent-credential.nix index 292fd32d..b38b5914 100644 --- a/nix/host-modules/glue-queue-agent-credential.nix +++ b/nix/host-modules/glue-queue-agent-credential.nix @@ -5,10 +5,10 @@ # value there" into a value. # # ⚠️ Two files, not one, and that is the consumer's shape rather than a -# preference. `swarm_queue_client::QueueConfig::from_env` takes the secret as -# a PATH (`_OIDC_CLIENT_SECRET_FILE`) and the client id as a VALUE -# (`_OIDC_CLIENT_ID`), so splitting them here is what lets the next -# slice hand both to an agent without parsing anything. +# preference. `swarm-logs` inside the agent reads the secret and the client id +# each from its own file (`_OIDC_CLIENT_SECRET_FILE`, +# `_OIDC_CLIENT_ID_FILE`), so splitting them here is what lets +# hive-c0re hand both to an agent without parsing anything. # # ⚠️ An ABSENT file means this hive's agents do not connect to the queue, and # that is correct rather than degraded. The publisher runs on the authelia @@ -259,9 +259,8 @@ in exit 0 fi - # Both or neither, for the reason `QueueConfig::from_env` refuses a - # half-set environment: a client that finds one of the two comes up - # "fine" and never connects. + # Both or neither: a client that finds one of the two comes up "fine" + # and never connects. if [ -z "$secret" ] || [ -z "$client_id" ]; then echo "swarm-bao returned an empty field of ${credentialPath}; leaving the files as they are." >&2 exit 0 diff --git a/nix/host-modules/hive-c0re/default.nix b/nix/host-modules/hive-c0re/default.nix index c43a7019..9d030f95 100644 --- a/nix/host-modules/hive-c0re/default.nix +++ b/nix/host-modules/hive-c0re/default.nix @@ -316,26 +316,18 @@ in # the daemon as well would put a secret on a process that has nowhere # to present it. LoadCredential = - # The swarm-queue client secret this hive authenticates with to - # publish its own status. `LoadCredential` and not a copy: root - # reads the plaintext at unit start and hive-core sees it 0400 - # under `%d`, so the secret never gains a second on-disk copy - # and the daemon never needs read access to wherever it lives. - # (The callout responder copies instead only because it - # delivers into a container, across a filesystem boundary.) - lib.optional - (config.services.hyperhive.deploy.hive-controller.statusPublish.clientSecretFile != null) - "swarm-status-client.secret:${config.services.hyperhive.deploy.hive-controller.statusPublish.clientSecretFile}" - # The secret store's client identity, on the same reasoning one - # paragraph up — with a sharper edge: ./glue-bao-tls.nix mints the - # key `0600` inside a `0700` directory, so hive-core cannot read it - # at all. Loading it as a credential is what makes the store - # reachable from an unprivileged daemon without widening either. + # The secret store's client identity. `LoadCredential` and not a + # copy: root reads it at unit start and hive-core sees it 0400 + # under `%d`, so it never gains a second on-disk copy. And + # ./glue-bao-tls.nix mints the key `0600` inside a `0700` + # directory, so hive-core cannot read the original at all. + # Loading it as a credential is what makes the store reachable + # from an unprivileged daemon without widening either. # The gate is the identity, never `deploy.bao.enable`: a hive that # reads a store on another machine holds one of these and runs no # store. ./glue-queue-agent-credential.nix gates its own reader the # same way. - ++ lib.optionals baoClientIdentity [ + lib.optionals baoClientIdentity [ "bao-client.pem:${baoDeploy.clientCertFile}" "bao-client-key.pem:${baoDeploy.clientKeyFile}" ] diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 04d30c59..cc875197 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -223,22 +223,20 @@ in } // # Swarm-queue coordinates for offering this hive's status upward - # (hive-c0re::swarm_status). All four together or none: a half-set + # (hive-c0re::swarm_status). All three together or none: a half-set # environment is a deployment bug the daemon refuses to treat as # "no queue configured", because the failure it would otherwise # produce is a hive that comes up fine and silently never reports. - # The rule that a secret needs the other two options is asserted at - # eval in ./../swarm.nix, so this can only ever emit a complete set. # - # ⚠️ Gated on the SECRET as well as the URL. The URL defaults to the - # queue's name on every hive, so on its own it says where the queue is, - # not that this hive publishes; the secret is that claim. Gated on the - # URL alone, a hive with no secret would be handed a secret path that - # `LoadCredential` in ./default.nix never fills. + # The client secret is not among them: the daemon reads it from the + # swarm's secret store under this hive's own certificate + # (`swarm_secret_client::queue::hive_client_path`), so a hive with no + # store identity has no way to publish and is handed nothing. lib.optionalAttrs ( config.services.hyperhive.deploy.hive-controller.statusPublish.natsUrl != null - && config.services.hyperhive.deploy.hive-controller.statusPublish.clientSecretFile != null + && config.services.hyperhive.swarm.statusPublish.tokenEndpoint != null + && haveBaoClientIdentity ) { HIVE_C0RE_NATS_URL = config.services.hyperhive.deploy.hive-controller.statusPublish.natsUrl; @@ -247,9 +245,6 @@ in # `swarm.hives` — the hive does not choose its own name here, it uses # the one the roster gave it. HIVE_C0RE_OIDC_CLIENT_ID = "hive-${config.services.hyperhive.hiveName}"; - # `%d` is systemd's credentials directory — see the LoadCredential in - # ./default.nix. The daemon reads a path, never a value. - HIVE_C0RE_OIDC_CLIENT_SECRET_FILE = "%d/swarm-status-client.secret"; } // { # Where ../glue-queue-agent-credential.nix lands the AGENTS' queue diff --git a/nix/host-modules/swarm-secret-publisher.nix b/nix/host-modules/swarm-secret-publisher.nix index ef67d5e0..42c754f6 100644 --- a/nix/host-modules/swarm-secret-publisher.nix +++ b/nix/host-modules/swarm-secret-publisher.nix @@ -1,7 +1,7 @@ # The unit that puts swarm-level secrets into the swarm's secret store, so -# whoever needs one can read it there: a hive its agents' credential, a -# service or the controller its own. It mints nothing: every value is an OIDC -# client secret authelia minted, and this unit copies it. +# whoever needs one can read it there: a hive its own and its agents' +# credentials, a service or the controller its own. It mints nothing: every +# value is an OIDC client secret authelia minted, and this unit copies it. # # ⚠️ "Whoever", including a reader on THIS host. A swarm service's secret goes # into the store even when the service runs beside authelia, because its @@ -57,6 +57,10 @@ let hive: "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}${hyperhiveCfg.swarm.authelia.agentClientSuffix}"; + # The hive's own client id, which hive-c0re publishes the hive's status with. + # Composed as ./swarm-authelia.nix composes it, for the reason above. + hiveClientId = hive: "${hyperhiveCfg.swarm.authelia.hiveClientPrefix}${hive}"; + # The swarm's own services, as opposed to its hives. One client for the whole # swarm rather than one per hive, so one value in the store rather than a copy # each: `swarm/services//oidc/client`. @@ -100,8 +104,9 @@ in description = '' Publish the OIDC client secrets this host mints into the swarm's secret store, so a hive that does not run authelia can read its - agents' credential and a swarm service or the controller can read its - own — from wherever it runs, this host included. + own and its agents' credentials and a swarm service or the + controller can read its own — from wherever it runs, this host + included. Defaults to whether this host mints the OIDC secrets, which is the only half of the question that is a property of *this* host. @@ -114,8 +119,9 @@ in identity's job (see `baoClientCertFile`), not this option's. Turning it off leaves every hive but this one without its agents' - credential, the swarm controller unable to start, the swarm's Grafana - without any login at all, its collector pushing unauthenticated, the + credential, every hive unable to publish its status, the swarm + controller unable to start, the swarm's Grafana without any login at + all, its collector pushing unauthenticated, the queue's responder unable to start, and the homeserver unable to start. Each secret has exactly one route and this is the producer's end of it, so the honest reason to set it false is a deployment @@ -223,6 +229,21 @@ in fi '') hiveNames} + # Each hive's own client, which that hive's hive-c0re reads under its + # own certificate. The nix half of + # `swarm_secret_client::queue::hive_client_path`. + ${lib.concatMapStringsSep "\n" (hive: '' + src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${hiveClientId hive}.secret"} + if [ -s "$src" ]; then + bao kv put ${lib.escapeShellArg "secret/swarm/hives/${hive}/oidc/client"} \ + value=@"$src" + published=$((published + 1)) + else + echo "no minted secret at $src yet; the path unit will re-run this" >&2 + skipped=$((skipped + 1)) + fi + '') hiveNames} + ${lib.concatMapStringsSep "\n" (id: '' src=${lib.escapeShellArg "${deployCfg.authelia.hostClientSecretDir}/${id}.secret"} if [ -s "$src" ]; then diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index a307bcf6..5c8f1661 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -112,17 +112,6 @@ let pinnedHives = lib.attrNames ( lib.filterAttrs (_: hive: hive.certFingerprint != null) swarmCfg.hives ); - - # Whether this hive's minted status secret is on THIS host, which is the - # one status-publishing coordinate that is a statement about this - # machine's disk. - # - # ⚠️ Neither the token endpoint nor the queue's URL is gated on it. Both - # are the swarm's one address: the endpoint derived from - # `swarm.authelia.url`, the queue from its name, which resolves on every - # hive. So they are correct wherever the queue and the IdP run, and do - # not ask where anything runs. - queueLocal = deployCfg.nats.enable && deployCfg.authelia.enable && cfg.hiveName != null; in { options.services.hyperhive.swarm.hives = lib.mkOption { @@ -379,53 +368,6 @@ in designed for it. ''; } - { - # Deliberately an assertion and not a silent "then publish - # nothing": a secret without the coordinates it is presented at is - # a config an operator believes is working, and its runtime failure - # mode is the expensive one — the daemon comes up fine, never - # connects, and the hive reads `never_reported` on a dashboard - # nobody is watching yet. - # - # It encodes a property of the code rather than an intended shape — - # hive-c0re genuinely cannot publish with two of three coordinates — - # which is the distinction the `serviceDomains'` comment at the top - # of this file was written about. - # - # The three coordinates live in two namespaces now: the token - # endpoint is the swarm's one address, the other two are this - # machine's. The message spells all three paths out, because an - # operator told only the option names would look for them under - # one prefix and find one of them. - # - # ⚠️ Asymmetric on purpose. The token endpoint and the queue's URL - # both default to the swarm's one address on every hive: they say - # where the IdP and the queue are, not that this hive publishes, so - # either without the secret is publishing off rather than a half-set - # trio. Only the client secret is a claim to be publishing, and it - # needs the other two to mean anything. - assertion = - deployCfg.hive-controller.statusPublish.clientSecretFile == null - || ( - deployCfg.hive-controller.statusPublish.natsUrl != null - && swarmCfg.statusPublish.tokenEndpoint != null - ); - message = '' - This hive has a status-publishing client secret but not the - coordinates to publish with it. - - Currently: - deploy.hive-controller.statusPublish.natsUrl - = ${toString deployCfg.hive-controller.statusPublish.natsUrl} - swarm.statusPublish.tokenEndpoint - = ${toString swarmCfg.statusPublish.tokenEndpoint} - deploy.hive-controller.statusPublish.clientSecretFile - = ${toString deployCfg.hive-controller.statusPublish.clientSecretFile} - - Set the missing ones to publish this hive's status to the - swarm, or set clientSecretFile to null to turn publishing off. - ''; - } (nameGuards.mustNotEqual { option = "services.hyperhive.swarm.hives"; names = hiveNames; @@ -508,13 +450,14 @@ in }; # How this hive reaches the swarm queue to offer its own status - # (hive-c0re's `swarm_status`). Three coordinates: the token endpoint - # below follows the swarm's IdP address wherever that is, and the two - # under `deploy.hive-controller` default from the local swarm services - # when this host runs them and are set by hand otherwise. + # (hive-c0re's `swarm_status`): the token endpoint below, which follows + # the swarm's IdP address wherever that is, and the queue's URL under + # `deploy.hive-controller`. The client secret is not an option: hive-c0re + # reads it from the swarm's secret store under this hive's own + # certificate (`deploy.bao.clientCertFile`). # - # There is no `enable`: three coordinates that are all set is the - # enable. An extra flag would let a hive be configured-but-off, which + # There is no `enable`: both coordinates set, plus a store identity, is + # the enable. An extra flag would let a hive be configured-but-off, which # is one more state to explain and one more way to be silently quiet. options.services.hyperhive.swarm.statusPublish = { tokenEndpoint = lib.mkOption { @@ -534,16 +477,10 @@ in }; # What stays above is the one thing every hive shares: the address of the - # swarm's token endpoint. The other two coordinates are this machine's — - # where the queue is *as seen from here*, and where its secret sits on this - # disk — so they hang off the daemon that uses them. `hive-controller` is - # the hive-c0re daemon's namespace, hence these live there rather than - # under a `deploy.statusPublish` of their own. - # - # ⚠️ The all-or-nothing assertion above now spans both namespaces. That is - # one service's own options either side of the split, not a service reaching - # into a foreign `deploy.*`; its message names all three paths so an operator - # is not told to set two options under a path that has one. + # swarm's token endpoint. The queue's URL is this machine's — where the + # queue is *as seen from here* — so it hangs off the daemon that uses it. + # `hive-controller` is the hive-c0re daemon's namespace, hence it lives + # there rather than under a `deploy.statusPublish` of its own. options.services.hyperhive.deploy.hive-controller.statusPublish = { natsUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; @@ -558,41 +495,14 @@ in DNS. TLS only, and by name, since the queue's certificate carries the name and no address. - A URL alone does not publish. - {option}`services.hyperhive.deploy.hive-controller.statusPublish.clientSecretFile` - is what turns publishing on. Without it this hive computes its own - readiness as always and offers it to nobody, and the swarm - controller reports it `never_reported`, which is the honest - reading. - ''; - }; - - clientSecretFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = - if queueLocal then - "${deployCfg.authelia.hostClientSecretDir}/hive-${cfg.hiveName}.secret" - else - null; - defaultText = lib.literalExpression ''"''${deploy.authelia.hostClientSecretDir}/hive-''${hiveName}.secret" when this host runs both the queue and the IdP, else null''; - example = "/var/lib/secrets/swarm-queue-client.secret"; - description = '' - Path to a file holding the plaintext client secret for this - hive's `hive-` identity. - - A path and not a value: a secret in the Nix store is world - readable, and one in the environment is readable by anything - that can open {file}`/proc//environ`. - - Setting it is what makes this hive publish its status; null - turns publishing off. - - Defaults to authelia's own minted secret when the IdP runs on - this host. On any other hive the secret has to get here somehow, - and the swarm does not distribute it — copy it out of the swarm - host's - {option}`services.hyperhive.deploy.authelia.hostClientSecretDir` - with whatever secret management this deployment already uses. + A URL alone does not publish. This hive also needs its own store + identity ({option}`services.hyperhive.deploy.bao.clientCertFile`), + under which hive-c0re reads its `hive-` client secret + from the swarm's secret store; `swarm-secret-publish`, on the host + that runs authelia, puts it there. Without an identity, or with + this set to null, this hive computes its own readiness as always + and offers it to nobody, and the swarm controller reports it + `never_reported`, which is the honest reading. ''; }; }; diff --git a/nix/module-eval/nats-tls.nix b/nix/module-eval/nats-tls.nix index 257843a7..3e08fead 100644 --- a/nix/module-eval/nats-tls.nix +++ b/nix/module-eval/nats-tls.nix @@ -80,26 +80,26 @@ let lib.recursiveUpdate remoteSecrets { deploy.swarm-controller.enable = true; } ); - # The same hive given its status secret by hand, which is what turns - # publishing on away from the IdP's host. - remotePublishing = hiveWithController ( - lib.recursiveUpdate remoteSecrets { - deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret"; - } - ); + # The same hive given its own store leaf by hand, which is what turns + # publishing on away from the IdP's host: hive-c0re reads the status secret + # from the store under it. The queue-credential reader's pair comes with it, + # because a hive that reads the store is refused at eval without one. + hiveLeaf = { + deploy.bao.clientCertFile = "/etc/pki/h1.pem"; + deploy.bao.clientKeyFile = "/etc/pki/h1-key.pem"; + deploy.bao.queueAgentClientCertFile = "/etc/pki/h1-queue-agent.pem"; + deploy.bao.queueAgentClientKeyFile = "/etc/pki/h1-queue-agent-key.pem"; + }; + remotePublishing = hiveWithController (lib.recursiveUpdate remoteSecrets hiveLeaf); - # A real half-config: the secret, with the URL it would be presented at - # taken away. - remoteSecretNoUrl = hive ( - lib.recursiveUpdate remoteSecrets { - deploy.hive-controller.statusPublish.clientSecretFile = "/var/lib/secrets/hive-h1.secret"; - deploy.hive-controller.statusPublish.natsUrl = null; - } + # The same leaf with the queue's URL taken away. + remoteLeafNoUrl = hive ( + lib.recursiveUpdate remoteSecrets ( + lib.recursiveUpdate hiveLeaf { deploy.hive-controller.statusPublish.natsUrl = null; } + ) ); failedAssertions = m: lib.filter (a: !a.assertion) m.assertions; - refusedStatusSecret = - m: lib.any (a: lib.hasInfix "status-publishing client secret" a.message) (failedAssertions m); policyScript = allLocal.systemd.services.swarm-bao-nats-tls-policy.script; leafUnit = allLocal.systemd.services.swarm-bao-nats-tls; @@ -318,44 +318,50 @@ let && remote.services.hyperhive.swarm.controller.queue.natsUrl == natsUrl; } { - # The URL is set and the secret is not, which is every such hive until - # an operator places one: publishing is off, not misconfigured. + # The URL is set and the hive holds no store leaf, which is every such + # hive until an operator places one: publishing is off, not + # misconfigured. name = "that hive evaluates without an assertion failure"; ok = failedAssertions remote == [ ]; } { - # Off means off: hive-c0re is handed no status coordinates, rather - # than a secret path nothing fills. - name = "without its status secret, that hive's hive-c0re is given no status coordinates"; - ok = - let - s = remote.systemd.services.hive-c0re; - in - !(s.environment ? HIVE_C0RE_NATS_URL) - && !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE) - && !(lib.any (lib.hasPrefix "swarm-status-client.secret:") ( - lib.toList (s.serviceConfig.LoadCredential or [ ]) - )); + # Off means off: hive-c0re is handed no status coordinates it could not + # fetch a secret for. + name = "without a store leaf, that hive's hive-c0re is given no status coordinates"; + ok = !(remote.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL); } { - # The secret alone turns publishing on, at the default URL. - name = "given its status secret, that hive publishes to the queue's name"; + # The leaf alone turns publishing on, at the default URL, with the + # secret read from the store rather than named as a file. + name = "given its store leaf, that hive publishes to the queue's name as hive-"; ok = let s = remotePublishing.systemd.services.hive-c0re; in failedAssertions remotePublishing == [ ] && s.environment.HIVE_C0RE_NATS_URL == natsUrl - && s.environment.HIVE_C0RE_OIDC_CLIENT_SECRET_FILE == "%d/swarm-status-client.secret" - && lib.elem "swarm-status-client.secret:/var/lib/secrets/hive-h1.secret" ( - lib.toList s.serviceConfig.LoadCredential - ); + && s.environment.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1" + && !(s.environment ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE); } { - # What the assertion was written for is still refused: a secret with - # nowhere to present it. - name = "a status secret without a queue URL is refused at eval"; - ok = refusedStatusSecret remoteSecretNoUrl && !(refusedStatusSecret remote); + # The host that runs the queue, the IdP and the store gets the same + # coordinates from the leaf ../host-modules/glue-bao-tls.nix mints there, + # and no file. + name = "the all-local hive publishes from the store too"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + in + e.HIVE_C0RE_NATS_URL == natsUrl + && e.HIVE_C0RE_OIDC_CLIENT_ID == "hive-h1" + && !(e ? HIVE_C0RE_OIDC_CLIENT_SECRET_FILE); + } + { + # A null URL turns publishing off. + name = "a hive with a store leaf and no queue URL is given no status coordinates"; + ok = + failedAssertions remoteLeafNoUrl == [ ] + && !(remoteLeafNoUrl.systemd.services.hive-c0re.environment ? HIVE_C0RE_NATS_URL); } ]; in diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix index 8d54ab63..1523b5d7 100644 --- a/nix/module-eval/secret-publisher.nix +++ b/nix/module-eval/secret-publisher.nix @@ -84,6 +84,24 @@ let deploy.matrix.enable = true; }; cases = [ + { + # Each hive's hive-c0re reads this exact path + # (`swarm_secret_client::queue::hive_client_path`, pinned by that crate's + # own test), for every hive in the roster, not only the one this host is. + # The agents' client beside it stays a separate path. + name = "the publisher writes every hive's own client under that hive's prefix"; + ok = + let + s = secretPublisherHere.systemd.services.swarm-secret-publish.script; + in + lib.all + (h: lib.hasInfix "secret/swarm/hives/${h}/oidc/client" s && lib.hasInfix "/hive-${h}.secret" s) + [ + "h1" + "h2" + ] + && lib.hasInfix "secret/swarm/hives/h1/queue/agent" s; + } { # The controller reads this exact path # (`swarm_secret_client::queue::controller_client_path`, pinned by that diff --git a/swarm-logs/src/auth.rs b/swarm-logs/src/auth.rs index 023c7548..236592b1 100644 --- a/swarm-logs/src/auth.rs +++ b/swarm-logs/src/auth.rs @@ -53,11 +53,11 @@ pub struct Config { impl Config { /// Read the configuration out of the environment the wrapper sets. /// - /// All four coordinates or none, for the reason - /// [`QueueConfig::from_env`] gives and `hive-agent`'s `swarm_queue` - /// repeats: a half-set environment produces a process that looks - /// configured and never authenticates. Unlike the harness, this binary - /// *errors* on the absent case rather than degrading — a CLI whose only + /// All four coordinates or none, for the reason `swarm-queue-client`'s + /// README gives and `hive-agent`'s `swarm_queue` repeats: a half-set + /// environment produces a process that looks configured and never + /// authenticates. Unlike the harness, this binary *errors* on the absent + /// case rather than degrading — a CLI whose only /// job is the query has nothing left to do without it, and an operator /// running it wants to be told why rather than handed an empty result /// they will read as "no logs matched". @@ -69,10 +69,9 @@ impl Config { let client_id_file = var("OIDC_CLIENT_ID_FILE"); let client_secret_file = var("OIDC_CLIENT_SECRET_FILE"); - // Outside the all-or-none group on purpose, exactly as in - // `QueueConfig::from_env`: a swarm behind a publicly-trusted - // certificate needs no extra anchor, so a CA path with no endpoint is - // meaningless rather than half-configured. Nothing in this tree sets + // Outside the all-or-none group on purpose: a swarm behind a + // publicly-trusted certificate needs no extra anchor, so a CA path + // with no endpoint is meaningless rather than half-configured. Nothing in this tree sets // it for an agent — a container already trusts the store's root, which // `hive_c0re::meta` embeds at build time. let ca_file = var("OIDC_CA_FILE").map(PathBuf::from); diff --git a/swarm-queue-client/README.md b/swarm-queue-client/README.md index 60754188..15cc1b69 100644 --- a/swarm-queue-client/README.md +++ b/swarm-queue-client/README.md @@ -32,24 +32,21 @@ nothing says so until someone reads a dashboard. ## Configuration -`QueueConfig::from_env(prefix)` reads `_NATS_URL`, -`_OIDC_TOKEN_ENDPOINT`, `_OIDC_CLIENT_ID` and -`_OIDC_CLIENT_SECRET_FILE`. - -The prefix is a parameter because the variables belong to the consuming unit — -a NixOS module sets them alongside its other options. What is shared is the -rule, not the spelling: **all four together or none at all.** A half-set -environment is a hard error, because the failure it would otherwise produce is -the expensive kind — the process comes up "fine", never connects, and the data -it was supposed to move silently stops. +A caller builds a `QueueConfig` itself from its own environment: the queue's +URL, the token endpoint and the client id, plus an optional CA file. The +variables belong to the consuming unit, so each daemon reads them under its own +prefix. What is shared is the rule, not the spelling: **the three coordinates +together or none at all.** A half-set environment is a hard error, because the +failure it would otherwise produce is the expensive kind — the process comes up +"fine", never connects, and the data it was supposed to move silently stops. The client secret is never in the environment, which would publish it to -anything that can read `/proc//environ`. `from_env` takes it as a +anything that can read `/proc//environ`. A caller passes it either as a **path** (`ClientSecret::File`), read per token request rather than cached, so -a rotation the operator believes took effect actually did. A caller that -fetched the secret from somewhere else builds the config itself with -`ClientSecret::Value`, held in memory; `swarm-controller` does this with the -secret it reads from the swarm's secret store. +a rotation the operator believes took effect actually did, or as a value it +fetched itself (`ClientSecret::Value`), held in memory. `swarm-controller` and +`hive-c0re` both do the latter with the secret they read from the swarm's +secret store. ## Token request shape: HTTP Basic, `audience`, `scope` diff --git a/swarm-queue-client/src/lib.rs b/swarm-queue-client/src/lib.rs index 7605f67a..2bc3df95 100644 --- a/swarm-queue-client/src/lib.rs +++ b/swarm-queue-client/src/lib.rs @@ -37,20 +37,13 @@ use std::path::PathBuf; /// `anyhow` — `?` converts for free, so nothing downstream is more verbose /// for it. The same split `hive-claude` uses. /// -/// The variants are the failures an operator acts on differently: a -/// half-configured environment is a deployment bug, a refused token is an +/// The variants are the failures an operator acts on differently: an +/// unreadable secret file is a deployment bug, a refused token is an /// identity-provider config problem, an unreachable queue is a network /// one. Collapsing them into one string would make that distinction a /// matter of reading prose. #[derive(Debug, thiserror::Error)] pub enum Error { - #[error( - "swarm queue is half-configured: {prefix}_NATS_URL, \ - {prefix}_OIDC_TOKEN_ENDPOINT, {prefix}_OIDC_CLIENT_ID and \ - {prefix}_OIDC_CLIENT_SECRET_FILE must be set together or not at all" - )] - PartialConfig { prefix: String }, - #[error("reading the queue client secret from {path}")] ClientSecret { path: String, @@ -519,56 +512,6 @@ impl ClientSecret { } } -impl QueueConfig { - /// Read the config from `_NATS_URL`, `_OIDC_TOKEN_ENDPOINT`, - /// `_OIDC_CLIENT_ID` and `_OIDC_CLIENT_SECRET_FILE`, or - /// `None` when the queue was not wired up for this deployment. - /// - /// The prefix is a parameter rather than a constant because the variables - /// belong to the *consuming unit* — a NixOS module sets them alongside its - /// other options, and two daemons sharing one name would be a worse - /// coupling than passing four characters. What is shared is the RULE - /// below, not the spelling. - /// - /// `None` rather than an error on purpose: a daemon serves its other - /// surfaces on hosts where the queue is not enabled, and refusing to start - /// there would trade a missing feature for a dead process. What must NOT - /// happen is a *half* configuration silently behaving like an absent one — - /// hence the explicit partial check below. - pub fn from_env(prefix: &str) -> Result, Error> { - let url = std::env::var(format!("{prefix}_NATS_URL")).ok(); - let token_endpoint = std::env::var(format!("{prefix}_OIDC_TOKEN_ENDPOINT")).ok(); - let client_id = std::env::var(format!("{prefix}_OIDC_CLIENT_ID")).ok(); - let secret = std::env::var(format!("{prefix}_OIDC_CLIENT_SECRET_FILE")).ok(); - - // Read outside the match on purpose: this one is INDEPENDENT of the - // all-or-none rule, so it must not participate in the tuple that - // decides whether the queue is configured at all. A CA path with no - // queue is meaningless rather than half-configured. - let ca_file = std::env::var(format!("{prefix}_OIDC_CA_FILE")) - .ok() - .map(PathBuf::from); - - match (url, token_endpoint, client_id, secret) { - (None, None, None, None) => Ok(None), - (Some(url), Some(token_endpoint), Some(client_id), Some(secret)) => Ok(Some(Self { - url, - token_endpoint, - client_id, - client_secret: ClientSecret::File(PathBuf::from(secret)), - ca_file, - })), - // A partially-set environment is a deployment bug, and the failure - // it would otherwise produce is the expensive kind: the process - // comes up "fine", never connects, and the data it was supposed to - // move silently stops moving. Naming the variables costs one line. - _ => Err(Error::PartialConfig { - prefix: prefix.to_owned(), - }), - } - } -} - /// Mint a fresh access token for the controller's own client. /// /// `client_credentials`, because there is no user here: the controller @@ -998,50 +941,6 @@ mod tests { .with_no_client_auth(); } - /// The all-unset case is the common one — most hosts do not run the queue. - /// - /// Uses a prefix no deployment sets, so it cannot pass vacuously by - /// running inside a configured environment (the guard below covers the - /// same ground, and both are cheap). - #[test] - fn an_absent_environment_is_not_an_error() { - for k in [ - "SWARM_QUEUE_TEST_NATS_URL", - "SWARM_QUEUE_TEST_OIDC_TOKEN_ENDPOINT", - "SWARM_QUEUE_TEST_OIDC_CLIENT_ID", - "SWARM_QUEUE_TEST_OIDC_CLIENT_SECRET_FILE", - ] { - assert!(std::env::var(k).is_err(), "{k} must be unset for this test"); - } - assert!( - QueueConfig::from_env("SWARM_QUEUE_TEST") - .expect("absent is not an error") - .is_none() - ); - } - - /// The half-set case is the one the rule exists for: a deployment bug that - /// would otherwise look exactly like "no queue configured". - /// - /// SAFETY: single-threaded mutation of a process env var under a prefix no - /// other test or deployment uses; removed before returning. - #[test] - fn a_half_set_environment_is_a_hard_error() { - unsafe { - std::env::set_var("SWARM_QUEUE_HALF_NATS_URL", "nats://127.0.0.1:4222"); - } - let err = QueueConfig::from_env("SWARM_QUEUE_HALF") - .expect_err("a partial set must not read as absent"); - let msg = format!("{err}"); - assert!( - msg.contains("SWARM_QUEUE_HALF_OIDC_CLIENT_ID"), - "the error must name the missing variables, got: {msg}" - ); - unsafe { - std::env::remove_var("SWARM_QUEUE_HALF_NATS_URL"); - } - } - #[test] fn a_held_secret_is_read_back_as_itself() { let secret = ClientSecret::Value("s3cret".to_owned()); diff --git a/swarm-secret-client/src/queue.rs b/swarm-secret-client/src/queue.rs index 9c03b269..845762b1 100644 --- a/swarm-secret-client/src/queue.rs +++ b/swarm-secret-client/src/queue.rs @@ -86,6 +86,30 @@ pub struct ControllerCredential { pub value: String, } +/// The path holding `hive`'s own OIDC client secret: the `hive-` +/// identity hive-c0re publishes the hive's status to the swarm queue with. +/// +/// Not [`agent_client_path`], which is the agents' client on the same hive. +/// Both sit under the hive's own prefix, which the hive's policy reads +/// ([`crate::policy::render`]). +/// +/// The nix half is `swarm-secret-publisher.nix`, which writes it. +/// +/// # Errors +/// [`Error::PathSegment`] when `hive` contains anything but `[A-Za-z0-9_-]`. +pub fn hive_client_path(hive: &str) -> Result { + let prefix = principal_prefix(Kind::Hive, hive)?; + Ok(format!("{prefix}/oidc/client")) +} + +/// What [`hive_client_path`] holds. No client id beside the value: the reader +/// is told its id by its own environment, as the controller is. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HiveClientCredential { + /// The secret itself, under the field name every kind here uses. + pub value: String, +} + /// What [`agent_queue_path`] holds: the secret, and the agent it proves. /// /// No hive: an agent's identity is not tied to one, and the subjects the @@ -213,6 +237,40 @@ mod tests { assert!(reads(&own), "{own} should be readable under {granted:?}"); } + /// Spelled out in full because `swarm-secret-publisher.nix` writes the + /// same string by hand. + #[test] + fn a_hives_own_client_lands_under_its_own_principal_prefix() { + assert_eq!( + hive_client_path("alpha").expect("a plain name is legal"), + "swarm/hives/alpha/oidc/client" + ); + } + + #[test] + fn a_hives_own_policy_reads_its_client_and_no_other_hives() { + let doc = crate::policy::render("alpha").expect("a plain name is legal"); + let granted: Vec<&str> = doc + .lines() + .filter_map(|l| l.strip_prefix("path \"secret/data/")) + .filter_map(|p| p.strip_suffix("*\" {")) + .collect(); + let reads = |path: &str| granted.iter().any(|g| path.starts_with(g)); + let own = hive_client_path("alpha").expect("legal"); + assert!(reads(&own), "{own} should be readable under {granted:?}"); + let other = hive_client_path("beta").expect("legal"); + assert!(!reads(&other), "{other} is readable under {granted:?}"); + } + + #[test] + fn the_hive_client_objects_field_name_is_the_one_the_publisher_writes() { + let json = serde_json::to_value(HiveClientCredential { + value: "s3cr3t".to_owned(), + }) + .expect("serialises"); + assert_eq!(json, serde_json::json!({ "value": "s3cr3t" })); + } + #[test] fn the_controller_objects_field_name_is_the_one_the_publisher_writes() { let json = serde_json::to_value(ControllerCredential {