diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index 6a991e6a..9b9106be 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -83,7 +83,7 @@ The issuer is the **secret store's `pki` mount**, not a bare self-signed leaf. ` **Why an issuer, not a bare leaf**: a bare self-signed leaf is its own trust anchor, so every regeneration is a new anchor every consumer must re-trust, and an agent's build sets its trust store once, before any leaf exists. Agents and federation peers trust the store's root once; leaf rotation never breaks them. -**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. The CI and forge containers and federating peers read the bundle. Agents fetch the root from the store themselves ([`swarm/ca.md`](../swarm/ca.md#distributing-the-root)). +**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. Agents (via `security.pki.certificateFiles`), the CI and forge containers and federating peers all read the bundle. **Why on by default**: matrix-dart-sdk (FluffyChat's SDK) fetches `https:///.well-known/matrix/client` and never falls back to plain http, so without TLS the browser client can't bootstrap. diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index ebd9b050..3019fd22 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -240,10 +240,14 @@ naming, because it's subtle and it bites at a distance: > the store's root.** The derivation reads `security.pki.certificateFiles` > inside itself; the root doesn't exist there. -Two consumers, both reading the root at runtime: +Two consumers, and only one of them is fine: -- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start. -- **The Matrix container gets a bind mount**, via the shared +- **`hive-tls.nix` covers agents.** It folds the root into this hive's + `trust-bundle.pem`, `glue-hive-tls-controllers.nix` hands hive-c0re that + path as `HIVE_TLS_CA_PATH`, and the meta-flake renderer embeds that one + file next to every agent's flake. The bundle is the runtime-to-build-time + bridge. +- **The Matrix container crosses the same bridge**, via the shared `lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner` use: the bundle is bind-mounted read-only into the container, and a oneshot concatenates it with the system CAs before tuwunel starts. diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index c95181ba..090acf8f 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -590,8 +590,8 @@ The store's `swarm-services` role issues them instead, from the root inside its certificate is the only anchor the swarm has. Each hive fetches the certificate from the store at boot, so nothing build-time can name it — the system reads `security.pki.certificateFiles`' value at build time, which makes it the wrong -tool here. Host containers get a bind-mounted bundle assembled at boot; an -agent fetches the root itself with its own store certificate. +tool here. Trust reaches containers through a bind-mounted bundle assembled at +boot instead. ## Adding a secret diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index b4201afb..597495fa 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -100,7 +100,15 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default(); let initial = !dir.join(".git").exists(); - if !initial && on_disk == new_flake { + // Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in + // lockstep with its host source so the build-time `certificateFiles` + // list render_flake emits always resolves. Empty when the hive has no + // trust bundle. + let (ca_files, ca_changed) = ca_embed_state(&dir); + + // Skip only when both the flake AND the embedded CA are unchanged — a + // CA rotation with an otherwise-identical flake must still re-commit. + if !initial && on_disk == new_flake && !ca_changed { return Ok(()); } @@ -122,9 +130,10 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { std::fs::write(&flake_path, &new_flake) .with_context(|| format!("write {}", flake_path.display()))?; - // Agents fetch the swarm root from the store at boot, so no CA file sits - // next to the flake; drop the ones earlier renders left behind. - let ca_touched = remove_stale_ca_files(&dir); + // Materialise the embedded CA list next to flake.nix + drop any stale + // CA file; `ca_touched` is every filename written or removed, staged + // for commit below. Public CA certs only; no private key is embedded. + let ca_touched = materialise_ca_files(&dir, &ca_files)?; // Reconcile topology.json against the live agent set — adds // newly-spawned agents and drops removed ones. Idempotent; when @@ -166,7 +175,8 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { // contain '/flake.nix'". Lock then commit once with both // flake.nix and flake.lock — single commit per change. git(&dir, &["add", "flake.nix"]).await?; - // Stage every stale CA file removed above. `git add ` stages a deletion when the path is + // Stage every embedded CA file we wrote or removed (trust bundle + + // stale peer CAs). `git add ` stages a deletion when the path is // tracked and now gone; best-effort so the never-tracked-and-absent case // (pathspec mismatch) is a harmless no-op. for name in &ca_touched { @@ -194,8 +204,9 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { .filter_map(|f| match f.as_str() { "flake.nix" => Some("flake"), "flake.lock" => Some("lock"), - // Only ever a removal; see `remove_stale_ca_files`. - f if is_stale_ca_name(f) => Some("stale-ca"), + "hive-ca.pem" => Some("hive-ca"), + // Only ever a REMOVAL now; see `is_embedded_ca_name`. + f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"), "topology.json" => Some("topology"), _ => None, }) @@ -383,9 +394,6 @@ const FORWARDED_VARS: &[&str] = &[ // hive's store environment wholesale — the address is the only part of it // an agent may have, the certificate beside it being the hive's own. "HIVE_AGENT_BAO_ADDR", - // The mount path of the store's services `pki` engine, whose root an agent - // fetches to trust the swarm. Forwarded with the address, under the same gate. - "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", // Where an agent's `swarm-logs` queries, computed by the host for the // same reason as the two above: it is a gateway address, and a container // cannot derive it from anything it holds. @@ -429,12 +437,6 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ // environment, so an agent whose option says no store and whose env says // otherwise would have a check that is simply not there. ("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"), - // Read at build time to bake the root-fetch URL into - // `nix/agent-modules/hive-ca.nix`'s unit. - ( - "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", - "services.hyperhive.agent.bao.servicesPkiMount", - ), // Read at build time to decide whether `swarm-logs` is installed at all — // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own // reason: an agent whose option says no log store and whose env says @@ -535,6 +537,30 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> { .collect() } +/// Filename the hive's own trust anchors are embedded under at the +/// meta-flake root — the only entry of the embedded-CA list +/// `render_flake` references in `security.pki.certificateFiles` (see +/// `embedded_ca_files`). +const HIVE_CA_FILE: &str = "hive-ca.pem"; + +/// Host path of the hive's TLS trust anchors, when self-signed TLS is +/// active. The hyperhive TLS module sets `HIVE_TLS_CA_PATH` in hive-c0re's env +/// (to `/trust-bundle.pem`) whenever the +/// gateway serves a leaf issued by the secret store's `pki` mount. The +/// file holds that mount's root, the one anchor every hive's gateway leaf +/// chains to, and is copied verbatim, which `security.pki.certificateFiles` +/// accepts; nothing here parses it. Returns `Some(path)` only when the var +/// is set AND the cert exists on disk — so render + write stay consistent +/// (we never emit a `certificateFiles` reference to a file we didn't +/// embed). Only the public cert is ever read here. +fn hive_ca_source() -> Option { + let path = std::env::var("HIVE_TLS_CA_PATH").ok()?; + if path.is_empty() || !std::path::Path::new(&path).is_file() { + return None; + } + Some(path) +} + /// Extra `OTEL_RESOURCE_ATTRIBUTES` entries for every agent, read off /// hive-c0re's own unit env /// (`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`). @@ -548,32 +574,105 @@ fn otel_extra_resource_attributes() -> Option { .filter(|v| !v.is_empty()) } -/// Remove the CA files earlier renders embedded next to `flake.nix` -/// (`hive-ca.pem`, and the per-peer `peer-ca-.pem` before it), returning -/// every filename removed so the caller stages the deletion. Nothing writes -/// one now: agents fetch the swarm root from the store at boot, so a file left -/// here is referenced by nothing. -fn remove_stale_ca_files(dir: &Path) -> Vec { - let mut removed = Vec::new(); +/// The ordered set of CA certs embedded next to the meta flake, as +/// `(filename, host_source_path)` — now just the hive's own trust +/// anchors as `hive-ca.pem`, when self-signed TLS is active. +/// +/// That single file carries the store's `pki` root (the hyperhive TLS +/// module writes it into the trust bundle), so every hive's gateway leaf +/// validates from it and nothing is embedded per peer. +/// +/// `render_flake` emits exactly these filenames into +/// `security.pki.certificateFiles` and `sync_agents` materialises them, +/// so the rendered reference and the embedded files always agree. +fn embedded_ca_files() -> Vec<(String, String)> { + let mut out = Vec::new(); + if let Some(p) = hive_ca_source() { + out.push((HIVE_CA_FILE.to_owned(), p)); + } + out +} + +/// Write each desired embedded CA file next to `flake.nix` and remove +/// any stale one (self-signed TLS turned off, or a leftover peer CA), +/// so the flake never references a file we didn't write. Returns every +/// filename written or removed, for the caller to stage. The public CA +/// certs only; no private key is ever embedded. +fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result> { + let desired: std::collections::HashSet<&str> = + ca_files.iter().map(|(n, _)| n.as_str()).collect(); + let mut touched: Vec = Vec::new(); if let Ok(entries) = std::fs::read_dir(dir) { for e in entries.flatten() { let fname = e.file_name(); let Some(name) = fname.to_str() else { continue }; - if is_stale_ca_name(name) && std::fs::remove_file(dir.join(name)).is_ok() { - removed.push(name.to_owned()); + if is_embedded_ca_name(name) && !desired.contains(name) { + let _ = std::fs::remove_file(dir.join(name)); + touched.push(name.to_owned()); } } } - removed + for (name, content) in ca_files { + let path = dir.join(name); + std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?; + touched.push(name.clone()); + } + Ok(touched) } -/// True for the filenames [`remove_stale_ca_files`] owns. -fn is_stale_ca_name(name: &str) -> bool { - name == "hive-ca.pem" - || (name.starts_with("peer-ca-") - && Path::new(name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("pem"))) +/// True for a filename the embedded-CA machinery owns. Lets +/// `sync_agents` find stale CA files to clean up (a CA dropped from +/// config) without touching unrelated meta files. +/// +/// ⚠️ Still matches `peer-ca-.pem`, which `embedded_ca_files` no +/// longer produces — deliberately. Cleanup is driven by +/// "recognised but not desired", so this arm is exactly what removes a +/// per-peer CA file still sitting at a meta root. Drop it and those files +/// are orphaned at every meta root forever, referenced by nothing and +/// cleaned by nobody. +fn is_embedded_ca_name(name: &str) -> bool { + name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name)) +} + +/// True when `name` ends in a `.pem` extension (case-insensitive). Split +/// out so the embedded-CA filename checks share one spelling and dodge +/// clippy's case-sensitive-extension lint. +fn has_pem_ext(name: &str) -> bool { + Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pem")) +} + +/// Embedded-CA state for the meta repo: `(desired_files, changed)`. +/// `desired_files` is `(filename, contents)` for every CA that should sit +/// next to flake.nix. `changed` is true when the on-disk set differs in +/// any way — the file's contents changed, a CA appeared, or a +/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is +/// no longer wanted (stale, to be removed). Drives both the re-commit +/// decision and the materialise/cleanup in `sync_agents`, so a CA +/// rotation re-commits even when the flake itself is byte-identical. +fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) { + let desired: Vec<(String, String)> = embedded_ca_files() + .into_iter() + .filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c))) + .collect(); + let desired_names: std::collections::HashSet<&str> = + desired.iter().map(|(n, _)| n.as_str()).collect(); + + let mut changed = desired.iter().any(|(name, content)| { + std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content + }); + + // A previously-embedded CA file no longer wanted → stale (removal is + // a change even when every desired file already matches on disk). + if !changed && let Ok(entries) = std::fs::read_dir(dir) { + changed = entries.flatten().any(|e| { + e.file_name() + .to_str() + .is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name)) + }); + } + (desired, changed) } /// Resolve every custom agent's [`AgentPin`] and return `agents` with it set. @@ -763,6 +862,26 @@ fn render_flake( " services.hyperhive.agent.claudeCodePath = \"{path}\";" ); } + // CA trust: embed the hive's trust bundle so each agent validates + // the store's `pki` root at build time (one file; see + // `hive_ca_source`). `certificateFiles` is build-time, so the certs + // travel with the flake source: `sync_agents` writes `./hive-ca.pem` + // next to flake.nix and stages it. Only public CA certs are embedded; + // no private key ever leaves the host. The filename list matches + // `sync_agents` exactly (both derive it from `embedded_ca_files`), so + // we never reference a file we didn't embed; emitted only when the + // list is non-empty. + let ca_refs: Vec = embedded_ca_files() + .into_iter() + .map(|(name, _)| format!("./{name}")) + .collect(); + if !ca_refs.is_empty() { + let _ = writeln!( + out, + " security.pki.certificateFiles = [ {} ];", + ca_refs.join(" ") + ); + } // Extra OTEL resource labels (`deploy.hive-controller.otel.*`) are the one // OTEL value rendered here; enablement and the collector endpoint are // agent-module defaults. @@ -1886,13 +2005,24 @@ mod tests { } #[test] - fn render_flake_embeds_no_ca() { - // Agents fetch the swarm root from the store at boot; the flake names - // no CA file whatever the host env says, including the variable - // hive-c0re used to embed from. + fn render_flake_embeds_hive_ca_when_signalled() { + // When the hyperhive TLS module signals a trust bundle via + // HIVE_TLS_CA_PATH (and the cert exists), the agent module must + // trust it at build time via security.pki.certificateFiles. Absent + // the signal, no reference is emitted (so the flake doesn't point at + // a file that was never embedded). // - // Serialised against every other env-mutating test in the crate. + // Serialised against every other env-mutating test in the crate. Taken + // before the fixture files so the guard covers the whole window in + // which HIVE_TLS_CA_PATH / HIVE_PEER_CA_PATHS are perturbed. let _env = crate::test_env::lock(); + let ca_file = std::env::temp_dir().join(format!("hive-ca-test-{}.pem", std::process::id())); + std::fs::write( + &ca_file, + "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", + ) + .expect("write temp CA"); + let render = || { render( "git+https://forge.example/hyperhive/hyperhive.git?rev=0123abcd", @@ -1901,59 +2031,70 @@ mod tests { &[sample_spec("alice", 9001)], ) }; + + // A leftover peer-CA file + the env var that names it. Both must be + // inert: the store's root rides in the hive's own trust bundle, so + // nothing per-peer is embedded. + let stale_peer = + std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id())); + std::fs::write( + &stale_peer, + "-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n", + ) + .expect("write stale peer CA"); + // SAFETY: serialised by the guard at the top of this test; restored // before returning. unsafe { - std::env::set_var("HIVE_TLS_CA_PATH", "/nonexistent/trust-bundle.pem"); + std::env::remove_var("HIVE_PEER_CA_PATHS"); + std::env::set_var("HIVE_TLS_CA_PATH", &ca_file); } - let flake = render(); + let with_ca = render(); + unsafe { + std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string()); + } + let with_stale_peer_env = render(); + // The stale var alone, with no trust bundle: must produce nothing. unsafe { std::env::remove_var("HIVE_TLS_CA_PATH"); } - // Presence control: the render does carry per-agent module config, so - // the absences below are not an empty render. + let stale_peer_env_only = render(); + unsafe { + std::env::remove_var("HIVE_PEER_CA_PATHS"); + } + let without_ca = render(); + let _ = std::fs::remove_file(&ca_file); + let _ = std::fs::remove_file(&stale_peer); + assert!( - flake.contains("services.hyperhive.agent."), - "per-agent module block missing:\n{flake}" + with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"), + "CA cert must be wired into certificateFiles when signalled:\n{with_ca}" ); - for needle in ["hive-ca.pem", "certificateFiles", "peer-ca-"] { - assert!( - !flake.contains(needle), - "the flake must not name {needle}:\n{flake}" - ); - } - } - - #[test] - fn stale_ca_files_are_removed_and_nothing_else() { - let dir = tempfile::tempdir().expect("tempdir"); - for name in ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM", "flake.nix"] { - std::fs::write(dir.path().join(name), "x").expect("write"); - } - // Not CA files this machinery owns, though they look similar. - for name in ["peer-ca-0.txt", "other-ca.pem"] { - std::fs::write(dir.path().join(name), "x").expect("write"); - } - let mut removed = remove_stale_ca_files(dir.path()); - removed.sort(); - assert_eq!(removed, ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM"]); - for kept in ["flake.nix", "peer-ca-0.txt", "other-ca.pem"] { - assert!(dir.path().join(kept).exists(), "{kept} must survive"); - } - assert!(remove_stale_ca_files(dir.path()).is_empty()); - } - - #[test] - fn services_pki_mount_is_forwarded_as_both_env_and_option() { - assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT")); - let mut out = String::new(); - push_forwarded_var_options( - &mut out, - &[("HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", "pki".to_string())], + // The regression guard, scoped to the CA list rather than the whole + // render. Comparing the two flakes wholesale looks stronger and is + // actually FLAKY: `cargo test` runs these in parallel threads and + // sibling tests mutate process env (OTEL, forge URLs) between the + // two `render()` calls, so a whole-output equality assertion fails + // on changes that have nothing to do with this test. + assert!( + with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"), + "HIVE_PEER_CA_PATHS must not change the CA list — the store's \ + root inside the hive's own trust bundle is the only anchor \ + embedded:\n{with_stale_peer_env}" ); - assert_eq!( - out, - " services.hyperhive.agent.bao.servicesPkiMount = \"pki\";\n" + assert!( + !with_stale_peer_env.contains("peer-ca-"), + "no peer-ca-.pem may be embedded, however HIVE_PEER_CA_PATHS \ + is set:\n{with_stale_peer_env}" + ); + assert!( + !stale_peer_env_only.contains("security.pki.certificateFiles"), + "a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \ + reference on its own:\n{stale_peer_env_only}" + ); + assert!( + !without_ca.contains("security.pki.certificateFiles"), + "no certificateFiles reference without any CA signal:\n{without_ca}" ); } diff --git a/hive-forge/Cargo.toml b/hive-forge/Cargo.toml index 1b6b6053..5491547a 100644 --- a/hive-forge/Cargo.toml +++ b/hive-forge/Cargo.toml @@ -26,8 +26,8 @@ reqwest = { workspace = true, features = [ # `rustls` carries the platform verifier, which reads the OS trust store # (and honours `SSL_CERT_FILE`), so hive-forge validates the hive # gateway's self-signed leaf once the store's root is in the agent's trust - # store (an agent fetches it from the secret store at boot and points - # `SSL_CERT_FILE` at it) while public CAs still validate. + # store — embedded into agent flakes via `security.pki.certificateFiles` + # — while public CAs still validate. # # The separate `rustls-native-certs` / `webpki-roots` features this entry # used to name were 0.13.1-era and no longer exist; naming a feature that diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index 5db3dbdc..b8949cc5 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -43,7 +43,6 @@ in ./frontend.nix ./github.nix ./github-token.nix - ./hive-ca.nix ./hive-facts.nix ./logs.nix ./matrix.nix diff --git a/nix/agent-modules/hive-ca.nix b/nix/agent-modules/hive-ca.nix deleted file mode 100644 index 94effef1..00000000 --- a/nix/agent-modules/hive-ca.nix +++ /dev/null @@ -1,196 +0,0 @@ -# The swarm's root certificate in this agent's trust, fetched from the store -# at boot by the agent itself. -# -# The root is the store's services `pki` mount's CA. It does not exist when the -# agent is built, so the meta flake names no CA file: this unit reads it from -# the store with the agent's own certificate (./bao.nix keeps it current) and -# writes two files, both public: -# -# - `bundle`: the system CAs plus the swarm root. `SSL_CERT_FILE` replaces the -# trust store of Rust (rustls platform verifier), Go and OpenSSL clients, so -# it carries both halves. -# - `root`: the swarm root alone, for `NODE_EXTRA_CA_CERTS`, which adds to -# Node's store (Claude Code). -# -# A process reads its trust when it starts, so a rotated root reaches the -# consumers below at their next start. They `Requires=` this unit, which keeps -# one from running with a store that lacks the root. -{ - pkgs, - lib, - config, - ... -}: -let - cfg = config.services.hyperhive.agent.bao; - agentCfg = config.services.hyperhive.agent; - - agentName = agentCfg.user.name; - - # The server CA id ./bao.nix loads. - serverCaCredential = "hive-agent-bao-server-ca"; - - unitName = "hive-agent-trust-bundle"; - runtimeDir = unitName; - bundleFile = "/run/${runtimeDir}/bundle.pem"; - rootFile = "/run/${runtimeDir}/root.pem"; - - systemBundle = "/etc/ssl/certs/ca-certificates.crt"; - - configured = cfg.addr != null; - - storeRetry = import ../host-modules/lib/store-retry.nix { }; - - # Units that dial a swarm service over TLS. The first group is defined - # whenever the agent module is; the rest exist under the same condition as - # their definitions, because ordering on a unit that is not defined would - # generate an inert one. - consumers = [ - "hive-agent" - "hive-bash-daemon" - "hive-subagent-daemon" - "hive-mcp-http" - "hive-forge-notify" - ] - ++ lib.optional (agentCfg.matrixAccounts != { }) "hive-matrix-daemon" - ++ lib.optional (agentCfg.github.enable) "hive-github-notify" - ++ lib.optional (agentCfg.icon != null && agentCfg.forge.url != null) "forge-avatar-sync"; - - trustEnv = { - SSL_CERT_FILE = bundleFile; - NODE_EXTRA_CA_CERTS = rootFile; - }; -in -{ - options.services.hyperhive.agent.bao.servicesPkiMount = lib.mkOption { - type = lib.types.str; - default = "pki"; - example = "pki"; - description = '' - Mount path of the store's services `pki` engine, whose root is the one - anchor every swarm service certificate chains to. Set by the generated - meta flake from the host's - `services.hyperhive.deploy.bao.servicesPkiMountPath`. - `${unitName}.service` reads `/ca/pem` from it. - ''; - }; - - config = lib.mkIf configured { - # Every process, including a bash task's children, resolves trust through - # these, as it resolves `HIVE_FORGE_TOKEN_FILE`. - systemd.globalEnvironment = trustEnv; - environment.variables = trustEnv; - - systemd.services = { - ${unitName} = { - description = "fetch the swarm root certificate from the secret store"; - after = [ - "network.target" - "hive-agent-bao-identity.service" - ]; - wantedBy = [ "multi-user.target" ]; - path = [ - pkgs.coreutils - pkgs.curl - pkgs.openssl - ]; - inherit (storeRetry) startLimitBurst startLimitIntervalSec; - serviceConfig = storeRetry.serviceConfig // { - Type = "oneshot"; - RemainAfterExit = false; - TimeoutStartSec = 60; - User = agentName; - Group = agentName; - RuntimeDirectory = runtimeDir; - # Public certificates, read by every consumer's user. - RuntimeDirectoryMode = "0755"; - RuntimeDirectoryPreserve = "yes"; - UMask = "0022"; - LoadCredential = [ serverCaCredential ]; - }; - # Overrides the global value: this unit runs before the bundle exists, - # and its own TLS is checked against the credential below or the - # system bundle. - environment = { - SSL_CERT_FILE = lib.mkForce systemBundle; - BAO_ADDR = cfg.addr; - BAO_CLIENT_CERT = cfg.clientCertFile; - BAO_CLIENT_KEY = cfg.clientKeyFile; - }; - script = '' - set -euo pipefail - - for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do - if [ ! -s "$f" ]; then - echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2 - exit 1 - fi - done - - cacert= - if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then - cacert="$CREDENTIALS_DIRECTORY/${serverCaCredential}" - echo "trusting bao via the delivered server CA credential" >&2 - else - echo "no bao server CA credential delivered; trusting the system bundle" >&2 - fi - - dir=/run/${runtimeDir} - err=$(mktemp -p "$dir" err.XXXXXX) - raw=$(mktemp -p "$dir" raw.XXXXXX) - root=$(mktemp -p "$dir" root.XXXXXX) - bundle=$(mktemp -p "$dir" bundle.XXXXXX) - trap 'rm -f "$err" "$raw" "$root" "$bundle"' EXIT - - # The listener demands a client certificate and the endpoint takes no - # token, so no login precedes this. - if ! curl --fail --silent --show-error \ - --connect-timeout 10 --max-time 30 \ - ''${cacert:+--cacert "$cacert"} \ - --cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \ - --output "$raw" \ - ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then - echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2 - cat "$err" >&2 - exit 1 - fi - - # Re-encoded rather than copied: it fails on a body that is not a - # certificate and ends the file in the newline the concatenation - # below relies on. - if ! openssl x509 -in "$raw" -out "$root" 2>"$err"; then - echo "the store's ${cfg.servicesPkiMount} root does not parse as a certificate." >&2 - cat "$err" >&2 - exit 1 - fi - - cat ${systemBundle} "$root" >"$bundle" - - # Neither file moves unless it changed, so a reader never sees a half - # write and the timer's rerun is silent. - if cmp -s "$bundle" ${bundleFile} && cmp -s "$root" ${rootFile}; then - echo "the store's ${cfg.servicesPkiMount} root is unchanged" - exit 0 - fi - chmod 0644 "$root" "$bundle" - mv -f "$root" ${rootFile} - mv -f "$bundle" ${bundleFile} - echo "wrote the swarm root certificate from the store's ${cfg.servicesPkiMount} mount" - ''; - }; - } - // lib.genAttrs consumers (_: { - after = [ "${unitName}.service" ]; - requires = [ "${unitName}.service" ]; - }); - - systemd.timers.${unitName} = { - description = "re-read the swarm root certificate from the secret store"; - wantedBy = [ "timers.target" ]; - timerConfig = { - OnUnitInactiveSec = "1h"; - RandomizedDelaySec = "1min"; - }; - }; - }; -} diff --git a/nix/host-modules/glue-hive-tls-controllers.nix b/nix/host-modules/glue-hive-tls-controllers.nix index 938095be..07c9041f 100644 --- a/nix/host-modules/glue-hive-tls-controllers.nix +++ b/nix/host-modules/glue-hive-tls-controllers.nix @@ -2,10 +2,13 @@ # # ONE PAIRING PER FILE — hive-c0re and the swarm controller ← the trust bundle # ./hive-tls.nix assembles. Deleting this leaves daemons that trust only the -# platform roots: the swarm-queue clients die at -# `invalid peer certificate: UnknownIssuer` against an authelia signed by the -# swarm CA. +# platform roots: hive-c0re's lifecycle cannot hand agents the bundle, and the +# swarm-queue clients die at `invalid peer certificate: UnknownIssuer` against +# an authelia signed by the swarm CA. # +# `HIVE_TLS_CA_PATH` signals the hive-c0re lifecycle that a trust bundle exists: +# it bind-mounts the file (read-only, public certs ONLY — never a key) into each +# agent container, and the meta flake wires the per-agent trust bundle. # `*_OIDC_CA_FILE` names the same anchor for the swarm-queue clients that mint a # token from authelia over TLS. `swarm-queue-client` builds a bare # `reqwest::Client`, so it trusts only the platform roots without it. Where the @@ -40,6 +43,7 @@ in lib.mkMerge [ (lib.mkIf deployCfg.hive-controller.enable { systemd.services.hive-c0re.environment = { + HIVE_TLS_CA_PATH = trustBundle; HIVE_C0RE_OIDC_CA_FILE = trustBundle; }; }) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 049553c7..aff9c44c 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -278,9 +278,6 @@ in # environment wholesale would inherit the hive's certificate paths with it, # and those name a credential no agent may present. HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; - # The mount whose root an agent fetches to trust the swarm's services; the - # same value ../hive-tls.nix reads the root from. - HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = baoDeploy.servicesPkiMountPath; # `%d`, not the paths themselves: the key is `0600` root-owned and this # daemon runs as hive-core, so it never gets read access to the original. # See the LoadCredential in ./default.nix. diff --git a/nix/module-eval/agent-forge-bao.nix b/nix/module-eval/agent-forge-bao.nix index 8f51ebc8..8f4ccb80 100644 --- a/nix/module-eval/agent-forge-bao.nix +++ b/nix/module-eval/agent-forge-bao.nix @@ -47,38 +47,6 @@ let in let cases = [ - { - # ../agent-modules/hive-ca.nix. Presence arm first (the unit, the - # timer, the trust variables, and every consumer ordered after the - # unit and requiring it), then the absence arm for an agent with no - # store, which has no root to fetch. - name = "an agent with a store address fetches the swarm root and its consumers wait for it"; - ok = - let - svc = agentForgeBao.systemd.services; - unit = "hive-agent-trust-bundle.service"; - bundle = "/run/hive-agent-trust-bundle/bundle.pem"; - consumers = [ - "hive-agent" - "hive-bash-daemon" - "hive-subagent-daemon" - "hive-mcp-http" - "hive-forge-notify" - "forge-avatar-sync" - ]; - in - svc ? hive-agent-trust-bundle - && agentForgeBao.systemd.timers ? hive-agent-trust-bundle - && lib.hasInfix "/v1/pki/ca/pem" svc.hive-agent-trust-bundle.script - && agentForgeBao.systemd.globalEnvironment.SSL_CERT_FILE == bundle - && - agentForgeBao.systemd.globalEnvironment.NODE_EXTRA_CA_CERTS - == "/run/hive-agent-trust-bundle/root.pem" - && lib.all (c: lib.elem unit svc.${c}.after && lib.elem unit svc.${c}.requires) consumers - && !(agentForgeNoBao.systemd.services ? hive-agent-trust-bundle) - && !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE) - && !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ])); - } { # The whole switch is the store address, as for the queue credential. name = "an agent with a store address fetches its forge token"; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index e6060888..5e8d1db9 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -317,12 +317,6 @@ let in e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR; } - { - name = "an agent is told the mount its swarm root is read from, only where it is told the store"; - ok = - allLocal.systemd.services.hive-c0re.environment.HIVE_AGENT_BAO_SERVICES_PKI_MOUNT == "pki" - && !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_SERVICES_PKI_MOUNT); - } { # A hive with no certificate of its own can collect no agent's identity, # so forwarding an address would name a store nothing in the container diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index 51ba656f..abab4d62 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -693,7 +693,10 @@ let let bundle = "/var/lib/hive-tls/trust-bundle.pem"; envOf = m: unit: m.systemd.services.${unit}.environment or { }; - hiveVars = [ "HIVE_C0RE_OIDC_CA_FILE" ]; + hiveVars = [ + "HIVE_TLS_CA_PATH" + "HIVE_C0RE_OIDC_CA_FILE" + ]; anyOf = m: unit: vars: lib.any (v: (envOf m unit) ? ${v}) vars;