diff --git a/docs/swarm.md b/docs/swarm.md index 4e0d6fb8..eb0d6b8d 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -40,25 +40,35 @@ and `qualify()` / `qualified_label()` semantics. ```nix services.hyperhive.swarm.peers = { "lab.example.com" = { }; # CA-trusted (Let's Encrypt etc.) - "edge.corp" = { certFingerprint = "sha256:…"; }; # self-signed TLS + "edge.corp" = { certFingerprint = "sha256:…"; }; # self-signed TLS, c0re peer checks only + "mesh.internal" = { caCert = ./mesh-ca.pem; }; # self-signed, trusted for matrix federation }; ``` -The attrset key is the peer's DNS domain. `certFingerprint` is -optional: +The attrset key is the peer's DNS domain. Two independent, optional +trust knobs — pick by what you need to trust: -- **Omitted / null** — the system CA bundle validates the peer's TLS - cert. Correct for peers with Let's Encrypt or any standard CA cert. -- **Set** (`"sha256:…"`) — pin a specific cert fingerprint. Use this - for peers whose self-signed TLS cert doesn't chain to a CA your - host trusts. - -`certFingerprint` scopes **only** to hive-c0re's own peer HTTPS checks -(the P33RS dashboard links and agent peer discovery below). It is -**not** consulted by matrix federation — tuwunel validates a peer's -federation certificate against the system CA bundle independently (see -*Matrix federation* below), so pinning a fingerprint here does nothing -for a self-signed matrix gateway cert. +- **`certFingerprint`** (`"sha256:…"`) — pin the peer's TLS *leaf* + fingerprint. Scopes **only** to hive-c0re's own peer HTTPS checks + (the P33RS dashboard links + agent peer discovery below). It is + **not** consulted by matrix federation — tuwunel validates a peer's + federation certificate against the system CA bundle independently + (see *Matrix federation* below), so a fingerprint pin does nothing + for a self-signed matrix cert. +- **`caCert`** (path to the peer's root CA PEM) — embeds that CA (at + build time, into the nix store — no runtime file on the host) and + trusts it **everywhere the hive's own internal CA is**: it rides + alongside `hive-ca.pem` in every agent's + `security.pki.certificateFiles` (via the meta-flake renderer) **and** + in the matrix container's trust bundle, so tuwunel validates the + peer's *federation* TLS when it chains to that CA. Trust stays + **inside the hive** (agents + the matrix container), never the host + system trust store. **This is the knob that unblocks federation with + a self-signed peer hive** — use it instead of `certFingerprint` when + you control the peer's CA. (It does not affect hive-c0re's own peer + HTTPS checks — those stay on `certFingerprint` / the system bundle.) +- **Both omitted** — the stock system CA bundle validates the peer + (correct for Let's Encrypt / any publicly-trusted peer). ### Fingerprint format @@ -114,13 +124,13 @@ environment and forwarded to agent containers. 3. **Matrix federation** — when `matrix.enable` is on, tuwunel federates with the peer's matrix server (discovered via the peer's `.well-known/matrix/server` delegation, which the gateway serves). - Federation validates the peer's TLS certificate against the - **system CA bundle** — independently of `certFingerprint`, which it - never consults. A self-signed gateway certificate therefore won't - federate even with a fingerprint pinned above: the peers need - CA-issued certs (ACME) or a shared private CA trusted on both - gateway hosts. See `docs/matrix.md` for federation firewall + TLS - requirements. + Federation validates the peer's TLS certificate against the matrix + **container's** trust bundle — independently of `certFingerprint`, + which it never consults. A self-signed gateway certificate therefore + won't federate unless the peer's root CA is trusted: set `caCert` + above (embeds the peer CA into the matrix container's trust bundle), + or give the peers CA-issued certs (ACME). See `docs/matrix.md` for + federation firewall + TLS requirements. ## Bilateral setup diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 1479b842..87ef7b5c 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -67,12 +67,12 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default(); let initial = !dir.join(".git").exists(); - // Hive CA embedding (self-signed TLS): keep `./hive-ca.pem` at the meta - // root in lockstep with the host CA so the build-time `certificateFiles` - // reference render_flake emits always resolves. `ca_desired` is empty - // when self-signed TLS isn't active (cert / ACME mode). - let ca_path = dir.join(HIVE_CA_FILE); - let (ca_desired, ca_changed) = hive_ca_state(&dir); + // Embedded-CA list (self-signed hive CA + peer CAs): keep the + // `./hive-ca.pem` / `./peer-ca-.pem` files at the meta root in + // lockstep with their host sources so the build-time `certificateFiles` + // list render_flake emits always resolves. Empty when neither a + // self-signed hive CA nor any peer CA is configured. + let (ca_files, ca_changed) = ca_embed_state(&dir); // Skip only when both the flake AND the embedded CA are unchanged — a // CA rotation with an otherwise-identical flake must still re-commit. @@ -98,15 +98,10 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { std::fs::write(&flake_path, &new_flake) .with_context(|| format!("write {}", flake_path.display()))?; - // Materialise (or drop) the embedded hive CA next to flake.nix. When - // self-signed TLS is off, `ca_desired` is empty and we remove any stale - // cert so the flake (which no longer references it) stays buildable. - if ca_desired.is_empty() { - let _ = std::fs::remove_file(&ca_path); - } else if ca_changed { - std::fs::write(&ca_path, &ca_desired) - .with_context(|| format!("write {}", ca_path.display()))?; - } + // Materialise the embedded CA list next to flake.nix + drop any stale + // CA file; `ca_touched` is every filename written or removed, staged + // for commit below. Public CA certs only; no private key is embedded. + let ca_touched = materialise_ca_files(&dir, &ca_files)?; // Reconcile topology.json against the live agent set — adds // entries for newly-spawned agents (default: manager as parent, @@ -148,11 +143,13 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { // contain '/flake.nix'". Lock then commit once with both // flake.nix and flake.lock — single commit per change. git(&dir, &["add", "flake.nix"]).await?; - // Stage the embedded hive CA — added/updated when self-signed TLS is on, - // or its deletion when it was just removed. `git add ` stages a - // deletion when the path is tracked and now gone; best-effort so the - // never-tracked-and-absent case (pathspec mismatch) is a harmless no-op. - let _ = git(&dir, &["add", "--", HIVE_CA_FILE]).await; + // Stage every embedded CA file we wrote or removed (hive CA + peer + // CAs). `git add ` stages a deletion when the path is tracked + // and now gone; best-effort so the never-tracked-and-absent case + // (pathspec mismatch) is a harmless no-op. + for name in &ca_touched { + let _ = git(&dir, &["add", "--", name]).await; + } // Stage topology.json on every sync (regenerated by reconcile // above when the agent set changed). git add is a no-op when the // file content is unchanged. @@ -200,6 +197,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { "flake.nix" => Some("flake"), "flake.lock" => Some("lock"), "hive-ca.pem" => Some("hive-ca"), + f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"), "topology.json" => Some("topology"), "capabilities.json" => Some("capabilities"), "tool-groups.json" => Some("tool-groups"), @@ -576,10 +574,10 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> { .collect() } -/// Filename the hive CA cert is embedded under at the meta-flake root. -/// `sync_agents` writes it and `render_flake` references `./hive-ca.pem` -/// in `security.pki.certificateFiles` so every agent trusts it at build -/// time. +/// Filename the hive's own self-signed CA cert is embedded under at the +/// meta-flake root. One entry of the embedded-CA list `render_flake` +/// references in `security.pki.certificateFiles` (see `embedded_ca_files`); +/// peer CAs sit alongside it as `peer-ca-.pem`. const HIVE_CA_FILE: &str = "hive-ca.pem"; /// Host path of the hive CA *certificate*, when self-signed TLS is active. @@ -597,17 +595,113 @@ fn hive_ca_source() -> Option { Some(path) } -/// Embedded-CA state for the meta repo: `(desired_contents, changed)`. -/// `desired_contents` is the host hive CA cert (empty when self-signed TLS -/// is inactive); `changed` is true when it differs from what's already -/// embedded at `/hive-ca.pem`, so a CA rotation re-commits even when -/// the flake itself is byte-identical. -fn hive_ca_state(dir: &std::path::Path) -> (String, bool) { - let on_disk = std::fs::read_to_string(dir.join(HIVE_CA_FILE)).unwrap_or_default(); - let desired = hive_ca_source() - .and_then(|p| std::fs::read_to_string(p).ok()) - .unwrap_or_default(); - let changed = desired != on_disk; +/// Host paths of peer-hive root CA certificates, from `HIVE_PEER_CA_PATHS` +/// (colon-separated; set by hive-c0re.nix from `swarm.peers..caCert`). +/// Each is embedded alongside the hive CA so a peer's CA is trusted +/// everywhere the hive's own internal CA is — i.e. by every agent. Empty +/// segments and paths that don't resolve to a file are dropped, so we +/// never reference a `certificateFiles` entry we couldn't embed. +fn peer_ca_sources() -> Vec { + let Ok(raw) = std::env::var("HIVE_PEER_CA_PATHS") else { + return Vec::new(); + }; + raw.split(':') + .map(str::trim) + .filter(|p| !p.is_empty() && std::path::Path::new(p).is_file()) + .map(ToOwned::to_owned) + .collect() +} + +/// The ordered set of CA certs embedded next to the meta flake, as +/// `(filename, host_source_path)`. The self-signed hive CA (when active) +/// is `hive-ca.pem`; each peer CA is `peer-ca-.pem` in declaration +/// order. `render_flake` emits exactly these filenames into +/// `security.pki.certificateFiles` and `sync_agents` materialises them, +/// so the rendered reference and the embedded files always agree. +fn embedded_ca_files() -> Vec<(String, String)> { + let mut out = Vec::new(); + if let Some(p) = hive_ca_source() { + out.push((HIVE_CA_FILE.to_owned(), p)); + } + for (i, p) in peer_ca_sources().into_iter().enumerate() { + out.push((format!("peer-ca-{i}.pem"), p)); + } + out +} + +/// Write each desired embedded CA file next to `flake.nix` and remove +/// any stale one (a hive CA turned off, or a peer dropped from config), +/// so the flake never references a file we didn't write. Returns every +/// filename written or removed, for the caller to stage. The public CA +/// certs only; no private key is ever embedded. +fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result> { + let desired: std::collections::HashSet<&str> = + ca_files.iter().map(|(n, _)| n.as_str()).collect(); + let mut touched: Vec = Vec::new(); + if let Ok(entries) = std::fs::read_dir(dir) { + for e in entries.flatten() { + let fname = e.file_name(); + let Some(name) = fname.to_str() else { continue }; + if is_embedded_ca_name(name) && !desired.contains(name) { + let _ = std::fs::remove_file(dir.join(name)); + touched.push(name.to_owned()); + } + } + } + for (name, content) in ca_files { + let path = dir.join(name); + std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?; + touched.push(name.clone()); + } + Ok(touched) +} + +/// True for a filename `embedded_ca_files` can produce — the hive CA or +/// a `peer-ca-.pem`. Lets `sync_agents` find stale CA files to clean +/// up (a CA dropped from config) without touching unrelated meta files. +fn is_embedded_ca_name(name: &str) -> bool { + name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name)) +} + +/// True when `name` ends in a `.pem` extension (case-insensitive). Split +/// out so the embedded-CA filename checks share one spelling and dodge +/// clippy's case-sensitive-extension lint. +fn has_pem_ext(name: &str) -> bool { + Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pem")) +} + +/// Embedded-CA state for the meta repo: `(desired_files, changed)`. +/// `desired_files` is `(filename, contents)` for every CA that should sit +/// next to flake.nix (the hive CA + each peer CA). `changed` is true when +/// the on-disk set differs in any way — a file's contents changed, a new +/// CA appeared, or a previously-embedded CA (`hive-ca.pem` / +/// `peer-ca-*.pem`) is no longer wanted (stale, to be removed). Drives +/// both the re-commit decision and the materialise/cleanup in +/// `sync_agents`, so a CA rotation or a peer-set change re-commits even +/// when the flake itself is byte-identical. +fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) { + let desired: Vec<(String, String)> = embedded_ca_files() + .into_iter() + .filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c))) + .collect(); + let desired_names: std::collections::HashSet<&str> = + desired.iter().map(|(n, _)| n.as_str()).collect(); + + let mut changed = desired.iter().any(|(name, content)| { + std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content + }); + + // A previously-embedded CA file no longer wanted → stale (removal is + // a change even when every desired file already matches on disk). + if !changed && let Ok(entries) = std::fs::read_dir(dir) { + changed = entries.flatten().any(|e| { + e.file_name() + .to_str() + .is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name)) + }); + } (desired, changed) } @@ -748,16 +842,26 @@ where { "#, ); - // Self-signed TLS trust: embed the hive CA so every agent validates the - // gateway's self-signed leaf at build time. `security.pki.certificateFiles` - // is build-time, so the CA travels with the flake source — `sync_agents` - // writes `./hive-ca.pem` next to flake.nix and stages it. Only the public - // CA cert is embedded; the private key never leaves the host. Emitted only - // when hive-tls.nix signalled a CA (HIVE_TLS_CA_PATH) and the cert exists, - // matching the write condition in `sync_agents` so we never reference a - // file we didn't embed. - if hive_ca_source().is_some() { - out.push_str(" security.pki.certificateFiles = [ ./hive-ca.pem ];\n"); + // CA trust: embed every hive-trusted CA so each agent validates them at + // build time. The list is the hive's own self-signed CA (when active) + // plus every peer-hive root CA (`swarm.peers..caCert`) — a peer CA is + // trusted everywhere the hive's own internal CA is. `certificateFiles` is + // build-time, so the certs travel with the flake source: `sync_agents` + // writes `./hive-ca.pem` + `./peer-ca-.pem` next to flake.nix and + // stages them. Only public CA certs are embedded; no private key ever + // leaves the host. The filename list matches `sync_agents` exactly (both + // derive it from `embedded_ca_files`), so we never reference a file we + // didn't embed; emitted only when the list is non-empty. + let ca_refs: Vec = embedded_ca_files() + .into_iter() + .map(|(name, _)| format!("./{name}")) + .collect(); + if !ca_refs.is_empty() { + let _ = writeln!( + out, + " security.pki.certificateFiles = [ {} ];", + ca_refs.join(" ") + ); } out.push_str( r#" # The harness service inside the container runs as a @@ -1228,23 +1332,64 @@ mod tests { ) }; + // Two peer-hive CA temp files for the list cases. + let peer0 = std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id())); + let peer1 = std::env::temp_dir().join(format!("peer-ca1-test-{}.pem", std::process::id())); + std::fs::write( + &peer0, + "-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n", + ) + .expect("write peer CA 0"); + std::fs::write( + &peer1, + "-----BEGIN CERTIFICATE-----\np1\n-----END CERTIFICATE-----\n", + ) + .expect("write peer CA 1"); + let peer_paths = format!("{}:{}", peer0.display(), peer1.display()); + + // All env mutations are serialised within this one test (no other + // test asserts on these vars), restored before returning. unsafe { + std::env::remove_var("HIVE_PEER_CA_PATHS"); std::env::set_var("HIVE_TLS_CA_PATH", &ca_file); } let with_ca = render(); + // Hive CA + peer CAs: the list carries all three, hive CA first. + unsafe { + std::env::set_var("HIVE_PEER_CA_PATHS", &peer_paths); + } + let with_peers = render(); + // Peers only (this hive on ACME, federating with self-signed peers). unsafe { std::env::remove_var("HIVE_TLS_CA_PATH"); } + let peers_only = render(); + unsafe { + std::env::remove_var("HIVE_PEER_CA_PATHS"); + } let without_ca = render(); let _ = std::fs::remove_file(&ca_file); + let _ = std::fs::remove_file(&peer0); + let _ = std::fs::remove_file(&peer1); assert!( with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"), "CA cert must be wired into certificateFiles when signalled:\n{with_ca}" ); + assert!( + with_peers.contains( + "security.pki.certificateFiles = [ ./hive-ca.pem ./peer-ca-0.pem ./peer-ca-1.pem ]" + ), + "hive CA + peer CAs must all appear in the certificateFiles list:\n{with_peers}" + ); + assert!( + peers_only + .contains("security.pki.certificateFiles = [ ./peer-ca-0.pem ./peer-ca-1.pem ]"), + "peer CAs must be trusted even when this hive has no self-signed CA:\n{peers_only}" + ); assert!( !without_ca.contains("security.pki.certificateFiles"), - "no certificateFiles reference without the HIVE_TLS_CA_PATH signal:\n{without_ca}" + "no certificateFiles reference without any CA signal:\n{without_ca}" ); } } diff --git a/nix/modules/hive-c0re.nix b/nix/modules/hive-c0re.nix index eeaf1c4b..4d440a35 100644 --- a/nix/modules/hive-c0re.nix +++ b/nix/modules/hive-c0re.nix @@ -213,6 +213,35 @@ in then strip the colons and prepend `sha256:`. A malformed value is ignored with a warning rather than weakening trust. See docs/swarm.md for the full recipe. + + Scopes only to hive-c0re's own peer HTTPS checks — it does + NOT help Matrix federation (tuwunel validates against its + container trust bundle). For a self-signed peer whose root + CA you want trusted hive-wide (every agent + Matrix + federation), set `caCert` below. + ''; + }; + + caCert = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "./peers/edge-ca.pem"; + description = '' + Path to this peer hive's root CA certificate (PEM). When + set, the CA is embedded (at build time, into the nix store + — no runtime file on the host) and trusted **everywhere the + hive's own internal CA is**: it rides alongside `hive-ca.pem` + in each agent's `security.pki.certificateFiles` (via the + meta-flake renderer), and is added to the Matrix homeserver + container's trust bundle so tuwunel validates *federation* + TLS from a self-signed peer hive whose cert chains to it. + This is the CA-trust path that `certFingerprint` + (leaf-pinning, c0re-only) can't cover, and is what unblocks + Matrix federation with a self-signed peer hive. Trust stays + inside the hive (agents + the Matrix container), never the + host system trust store. Mutually complementary with + `certFingerprint`; set `caCert` for the federation case. See + docs/swarm.md. ''; }; @@ -825,7 +854,26 @@ in } ) config.services.hyperhive.swarm.peers ); - }; + } + // + lib.optionalAttrs + (lib.any (p: p.caCert != null) (lib.attrValues config.services.hyperhive.swarm.peers)) + { + # Peer-hive root CA file paths (colon-joined), one per peer that + # declares `swarm.peers..caCert`. hive-c0re's meta-flake + # renderer (meta.rs) embeds each next to every agent's flake and + # adds it to `security.pki.certificateFiles`, so a peer CA is + # trusted everywhere the hive's own internal CA (`hive-ca.pem`) + # is — i.e. by every agent. The matrix container trusts the same + # CAs separately for federation TLS. The `caCert` files are + # copied into the nix store at build, so these are store paths — + # nothing mutable lives on the host. + HIVE_PEER_CA_PATHS = lib.concatStringsSep ":" ( + lib.filter (c: c != null) ( + lib.mapAttrsToList (_domain: p: p.caCert) config.services.hyperhive.swarm.peers + ) + ); + }; serviceConfig = { ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --config ${serveConfig}"; # Migrate hive-c0re's *own* state to the service user after an diff --git a/nix/modules/hive-matrix.nix b/nix/modules/hive-matrix.nix index 555b8d4f..838bd08e 100644 --- a/nix/modules/hive-matrix.nix +++ b/nix/modules/hive-matrix.nix @@ -358,6 +358,21 @@ in { system.stateVersion = "26.05"; + # Peer-hive root CAs (`swarm.peers..caCert`) added to THIS + # container's trust bundle so tuwunel validates *federation* TLS + # from a self-signed peer hive (it checks the peer's federation + # cert against its trust bundle). Peer CAs are trusted everywhere + # the hive's own internal CA is — agents get them via the + # meta-flake renderer (`HIVE_PEER_CA_PATHS` → each agent's + # `security.pki.certificateFiles`); this block is the matrix + # container's copy, since the host `security.pki` store doesn't + # cross the container boundary. They are never installed in the + # HOST trust store. Null entries (CA-bundle / fingerprint-pinned + # peers) drop out. + security.pki.certificateFiles = lib.filter (c: c != null) ( + lib.mapAttrsToList (_domain: p: p.caCert) config.services.hyperhive.swarm.peers + ); + # tuwunel hard-fails to boot if `/etc/resolv.conf` has no # `nameserver` line (`Failed to configure DNS resolver ... no # nameservers found in config` → exit 1). This declarative