diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index 9b9106be..6a991e6a 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -83,7 +83,7 @@ The issuer is the **secret store's `pki` mount**, not a bare self-signed leaf. ` **Why an issuer, not a bare leaf**: a bare self-signed leaf is its own trust anchor, so every regeneration is a new anchor every consumer must re-trust, and an agent's build sets its trust store once, before any leaf exists. Agents and federation peers trust the store's root once; leaf rotation never breaks them. -**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. Agents (via `security.pki.certificateFiles`), the CI and forge containers and federating peers all read the bundle. +**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. The CI and forge containers and federating peers read the bundle. Agents fetch the root from the store themselves ([`swarm/ca.md`](../swarm/ca.md#distributing-the-root)). **Why on by default**: matrix-dart-sdk (FluffyChat's SDK) fetches `https:///.well-known/matrix/client` and never falls back to plain http, so without TLS the browser client can't bootstrap. diff --git a/docs/swarm/ca.md b/docs/swarm/ca.md index 3019fd22..ebd9b050 100644 --- a/docs/swarm/ca.md +++ b/docs/swarm/ca.md @@ -240,14 +240,10 @@ naming, because it's subtle and it bites at a distance: > the store's root.** The derivation reads `security.pki.certificateFiles` > inside itself; the root doesn't exist there. -Two consumers, and only one of them is fine: +Two consumers, both reading the root at runtime: -- **`hive-tls.nix` covers agents.** It folds the root into this hive's - `trust-bundle.pem`, `glue-hive-tls-controllers.nix` hands hive-c0re that - path as `HIVE_TLS_CA_PATH`, and the meta-flake renderer embeds that one - file next to every agent's flake. The bundle is the runtime-to-build-time - bridge. -- **The Matrix container crosses the same bridge**, via the shared +- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start. +- **The Matrix container gets a bind mount**, via the shared `lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner` use: the bundle is bind-mounted read-only into the container, and a oneshot concatenates it with the system CAs before tuwunel starts. diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 090acf8f..c95181ba 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -590,8 +590,8 @@ The store's `swarm-services` role issues them instead, from the root inside its certificate is the only anchor the swarm has. Each hive fetches the certificate from the store at boot, so nothing build-time can name it — the system reads `security.pki.certificateFiles`' value at build time, which makes it the wrong -tool here. Trust reaches containers through a bind-mounted bundle assembled at -boot instead. +tool here. Host containers get a bind-mounted bundle assembled at boot; an +agent fetches the root itself with its own store certificate. ## Adding a secret diff --git a/hive-c0re/src/meta.rs b/hive-c0re/src/meta.rs index 597495fa..b4201afb 100644 --- a/hive-c0re/src/meta.rs +++ b/hive-c0re/src/meta.rs @@ -100,15 +100,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default(); let initial = !dir.join(".git").exists(); - // Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in - // lockstep with its host source so the build-time `certificateFiles` - // list render_flake emits always resolves. Empty when the hive has no - // trust bundle. - let (ca_files, ca_changed) = ca_embed_state(&dir); - - // Skip only when both the flake AND the embedded CA are unchanged — a - // CA rotation with an otherwise-identical flake must still re-commit. - if !initial && on_disk == new_flake && !ca_changed { + if !initial && on_disk == new_flake { return Ok(()); } @@ -130,10 +122,9 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { std::fs::write(&flake_path, &new_flake) .with_context(|| format!("write {}", flake_path.display()))?; - // Materialise the embedded CA list next to flake.nix + drop any stale - // CA file; `ca_touched` is every filename written or removed, staged - // for commit below. Public CA certs only; no private key is embedded. - let ca_touched = materialise_ca_files(&dir, &ca_files)?; + // Agents fetch the swarm root from the store at boot, so no CA file sits + // next to the flake; drop the ones earlier renders left behind. + let ca_touched = remove_stale_ca_files(&dir); // Reconcile topology.json against the live agent set — adds // newly-spawned agents and drops removed ones. Idempotent; when @@ -175,8 +166,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { // contain '/flake.nix'". Lock then commit once with both // flake.nix and flake.lock — single commit per change. git(&dir, &["add", "flake.nix"]).await?; - // Stage every embedded CA file we wrote or removed (trust bundle + - // stale peer CAs). `git add ` stages a deletion when the path is + // Stage every stale CA file removed above. `git add ` stages a deletion when the path is // tracked and now gone; best-effort so the never-tracked-and-absent case // (pathspec mismatch) is a harmless no-op. for name in &ca_touched { @@ -204,9 +194,8 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> { .filter_map(|f| match f.as_str() { "flake.nix" => Some("flake"), "flake.lock" => Some("lock"), - "hive-ca.pem" => Some("hive-ca"), - // Only ever a REMOVAL now; see `is_embedded_ca_name`. - f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"), + // Only ever a removal; see `remove_stale_ca_files`. + f if is_stale_ca_name(f) => Some("stale-ca"), "topology.json" => Some("topology"), _ => None, }) @@ -394,6 +383,9 @@ const FORWARDED_VARS: &[&str] = &[ // hive's store environment wholesale — the address is the only part of it // an agent may have, the certificate beside it being the hive's own. "HIVE_AGENT_BAO_ADDR", + // The mount path of the store's services `pki` engine, whose root an agent + // fetches to trust the swarm. Forwarded with the address, under the same gate. + "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", // Where an agent's `swarm-logs` queries, computed by the host for the // same reason as the two above: it is a gateway address, and a container // cannot derive it from anything it holds. @@ -437,6 +429,12 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[ // environment, so an agent whose option says no store and whose env says // otherwise would have a check that is simply not there. ("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"), + // Read at build time to bake the root-fetch URL into + // `nix/agent-modules/hive-ca.nix`'s unit. + ( + "HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", + "services.hyperhive.agent.bao.servicesPkiMount", + ), // Read at build time to decide whether `swarm-logs` is installed at all — // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own // reason: an agent whose option says no log store and whose env says @@ -537,30 +535,6 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> { .collect() } -/// Filename the hive's own trust anchors are embedded under at the -/// meta-flake root — the only entry of the embedded-CA list -/// `render_flake` references in `security.pki.certificateFiles` (see -/// `embedded_ca_files`). -const HIVE_CA_FILE: &str = "hive-ca.pem"; - -/// Host path of the hive's TLS trust anchors, when self-signed TLS is -/// active. The hyperhive TLS module sets `HIVE_TLS_CA_PATH` in hive-c0re's env -/// (to `/trust-bundle.pem`) whenever the -/// gateway serves a leaf issued by the secret store's `pki` mount. The -/// file holds that mount's root, the one anchor every hive's gateway leaf -/// chains to, and is copied verbatim, which `security.pki.certificateFiles` -/// accepts; nothing here parses it. Returns `Some(path)` only when the var -/// is set AND the cert exists on disk — so render + write stay consistent -/// (we never emit a `certificateFiles` reference to a file we didn't -/// embed). Only the public cert is ever read here. -fn hive_ca_source() -> Option { - let path = std::env::var("HIVE_TLS_CA_PATH").ok()?; - if path.is_empty() || !std::path::Path::new(&path).is_file() { - return None; - } - Some(path) -} - /// Extra `OTEL_RESOURCE_ATTRIBUTES` entries for every agent, read off /// hive-c0re's own unit env /// (`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`). @@ -574,105 +548,32 @@ fn otel_extra_resource_attributes() -> Option { .filter(|v| !v.is_empty()) } -/// The ordered set of CA certs embedded next to the meta flake, as -/// `(filename, host_source_path)` — now just the hive's own trust -/// anchors as `hive-ca.pem`, when self-signed TLS is active. -/// -/// That single file carries the store's `pki` root (the hyperhive TLS -/// module writes it into the trust bundle), so every hive's gateway leaf -/// validates from it and nothing is embedded per peer. -/// -/// `render_flake` emits exactly these filenames into -/// `security.pki.certificateFiles` and `sync_agents` materialises them, -/// so the rendered reference and the embedded files always agree. -fn embedded_ca_files() -> Vec<(String, String)> { - let mut out = Vec::new(); - if let Some(p) = hive_ca_source() { - out.push((HIVE_CA_FILE.to_owned(), p)); - } - out -} - -/// Write each desired embedded CA file next to `flake.nix` and remove -/// any stale one (self-signed TLS turned off, or a leftover peer CA), -/// so the flake never references a file we didn't write. Returns every -/// filename written or removed, for the caller to stage. The public CA -/// certs only; no private key is ever embedded. -fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result> { - let desired: std::collections::HashSet<&str> = - ca_files.iter().map(|(n, _)| n.as_str()).collect(); - let mut touched: Vec = Vec::new(); +/// Remove the CA files earlier renders embedded next to `flake.nix` +/// (`hive-ca.pem`, and the per-peer `peer-ca-.pem` before it), returning +/// every filename removed so the caller stages the deletion. Nothing writes +/// one now: agents fetch the swarm root from the store at boot, so a file left +/// here is referenced by nothing. +fn remove_stale_ca_files(dir: &Path) -> Vec { + let mut removed = Vec::new(); if let Ok(entries) = std::fs::read_dir(dir) { for e in entries.flatten() { let fname = e.file_name(); let Some(name) = fname.to_str() else { continue }; - if is_embedded_ca_name(name) && !desired.contains(name) { - let _ = std::fs::remove_file(dir.join(name)); - touched.push(name.to_owned()); + if is_stale_ca_name(name) && std::fs::remove_file(dir.join(name)).is_ok() { + removed.push(name.to_owned()); } } } - for (name, content) in ca_files { - let path = dir.join(name); - std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?; - touched.push(name.clone()); - } - Ok(touched) + removed } -/// True for a filename the embedded-CA machinery owns. Lets -/// `sync_agents` find stale CA files to clean up (a CA dropped from -/// config) without touching unrelated meta files. -/// -/// ⚠️ Still matches `peer-ca-.pem`, which `embedded_ca_files` no -/// longer produces — deliberately. Cleanup is driven by -/// "recognised but not desired", so this arm is exactly what removes a -/// per-peer CA file still sitting at a meta root. Drop it and those files -/// are orphaned at every meta root forever, referenced by nothing and -/// cleaned by nobody. -fn is_embedded_ca_name(name: &str) -> bool { - name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name)) -} - -/// True when `name` ends in a `.pem` extension (case-insensitive). Split -/// out so the embedded-CA filename checks share one spelling and dodge -/// clippy's case-sensitive-extension lint. -fn has_pem_ext(name: &str) -> bool { - Path::new(name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("pem")) -} - -/// Embedded-CA state for the meta repo: `(desired_files, changed)`. -/// `desired_files` is `(filename, contents)` for every CA that should sit -/// next to flake.nix. `changed` is true when the on-disk set differs in -/// any way — the file's contents changed, a CA appeared, or a -/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is -/// no longer wanted (stale, to be removed). Drives both the re-commit -/// decision and the materialise/cleanup in `sync_agents`, so a CA -/// rotation re-commits even when the flake itself is byte-identical. -fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) { - let desired: Vec<(String, String)> = embedded_ca_files() - .into_iter() - .filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c))) - .collect(); - let desired_names: std::collections::HashSet<&str> = - desired.iter().map(|(n, _)| n.as_str()).collect(); - - let mut changed = desired.iter().any(|(name, content)| { - std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content - }); - - // A previously-embedded CA file no longer wanted → stale (removal is - // a change even when every desired file already matches on disk). - if !changed && let Ok(entries) = std::fs::read_dir(dir) { - changed = entries.flatten().any(|e| { - e.file_name() - .to_str() - .is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name)) - }); - } - (desired, changed) +/// True for the filenames [`remove_stale_ca_files`] owns. +fn is_stale_ca_name(name: &str) -> bool { + name == "hive-ca.pem" + || (name.starts_with("peer-ca-") + && Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("pem"))) } /// Resolve every custom agent's [`AgentPin`] and return `agents` with it set. @@ -862,26 +763,6 @@ fn render_flake( " services.hyperhive.agent.claudeCodePath = \"{path}\";" ); } - // CA trust: embed the hive's trust bundle so each agent validates - // the store's `pki` root at build time (one file; see - // `hive_ca_source`). `certificateFiles` is build-time, so the certs - // travel with the flake source: `sync_agents` writes `./hive-ca.pem` - // next to flake.nix and stages it. Only public CA certs are embedded; - // no private key ever leaves the host. The filename list matches - // `sync_agents` exactly (both derive it from `embedded_ca_files`), so - // we never reference a file we didn't embed; emitted only when the - // list is non-empty. - let ca_refs: Vec = embedded_ca_files() - .into_iter() - .map(|(name, _)| format!("./{name}")) - .collect(); - if !ca_refs.is_empty() { - let _ = writeln!( - out, - " security.pki.certificateFiles = [ {} ];", - ca_refs.join(" ") - ); - } // Extra OTEL resource labels (`deploy.hive-controller.otel.*`) are the one // OTEL value rendered here; enablement and the collector endpoint are // agent-module defaults. @@ -2005,24 +1886,13 @@ mod tests { } #[test] - fn render_flake_embeds_hive_ca_when_signalled() { - // When the hyperhive TLS module signals a trust bundle via - // HIVE_TLS_CA_PATH (and the cert exists), the agent module must - // trust it at build time via security.pki.certificateFiles. Absent - // the signal, no reference is emitted (so the flake doesn't point at - // a file that was never embedded). + fn render_flake_embeds_no_ca() { + // Agents fetch the swarm root from the store at boot; the flake names + // no CA file whatever the host env says, including the variable + // hive-c0re used to embed from. // - // Serialised against every other env-mutating test in the crate. Taken - // before the fixture files so the guard covers the whole window in - // which HIVE_TLS_CA_PATH / HIVE_PEER_CA_PATHS are perturbed. + // Serialised against every other env-mutating test in the crate. let _env = crate::test_env::lock(); - let ca_file = std::env::temp_dir().join(format!("hive-ca-test-{}.pem", std::process::id())); - std::fs::write( - &ca_file, - "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n", - ) - .expect("write temp CA"); - let render = || { render( "git+https://forge.example/hyperhive/hyperhive.git?rev=0123abcd", @@ -2031,70 +1901,59 @@ mod tests { &[sample_spec("alice", 9001)], ) }; - - // A leftover peer-CA file + the env var that names it. Both must be - // inert: the store's root rides in the hive's own trust bundle, so - // nothing per-peer is embedded. - let stale_peer = - std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id())); - std::fs::write( - &stale_peer, - "-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n", - ) - .expect("write stale peer CA"); - // SAFETY: serialised by the guard at the top of this test; restored // before returning. unsafe { - std::env::remove_var("HIVE_PEER_CA_PATHS"); - std::env::set_var("HIVE_TLS_CA_PATH", &ca_file); + std::env::set_var("HIVE_TLS_CA_PATH", "/nonexistent/trust-bundle.pem"); } - let with_ca = render(); - unsafe { - std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string()); - } - let with_stale_peer_env = render(); - // The stale var alone, with no trust bundle: must produce nothing. + let flake = render(); unsafe { std::env::remove_var("HIVE_TLS_CA_PATH"); } - let stale_peer_env_only = render(); - unsafe { - std::env::remove_var("HIVE_PEER_CA_PATHS"); + // Presence control: the render does carry per-agent module config, so + // the absences below are not an empty render. + assert!( + flake.contains("services.hyperhive.agent."), + "per-agent module block missing:\n{flake}" + ); + for needle in ["hive-ca.pem", "certificateFiles", "peer-ca-"] { + assert!( + !flake.contains(needle), + "the flake must not name {needle}:\n{flake}" + ); } - let without_ca = render(); - let _ = std::fs::remove_file(&ca_file); - let _ = std::fs::remove_file(&stale_peer); + } - assert!( - with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"), - "CA cert must be wired into certificateFiles when signalled:\n{with_ca}" + #[test] + fn stale_ca_files_are_removed_and_nothing_else() { + let dir = tempfile::tempdir().expect("tempdir"); + for name in ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM", "flake.nix"] { + std::fs::write(dir.path().join(name), "x").expect("write"); + } + // Not CA files this machinery owns, though they look similar. + for name in ["peer-ca-0.txt", "other-ca.pem"] { + std::fs::write(dir.path().join(name), "x").expect("write"); + } + let mut removed = remove_stale_ca_files(dir.path()); + removed.sort(); + assert_eq!(removed, ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM"]); + for kept in ["flake.nix", "peer-ca-0.txt", "other-ca.pem"] { + assert!(dir.path().join(kept).exists(), "{kept} must survive"); + } + assert!(remove_stale_ca_files(dir.path()).is_empty()); + } + + #[test] + fn services_pki_mount_is_forwarded_as_both_env_and_option() { + assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT")); + let mut out = String::new(); + push_forwarded_var_options( + &mut out, + &[("HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", "pki".to_string())], ); - // The regression guard, scoped to the CA list rather than the whole - // render. Comparing the two flakes wholesale looks stronger and is - // actually FLAKY: `cargo test` runs these in parallel threads and - // sibling tests mutate process env (OTEL, forge URLs) between the - // two `render()` calls, so a whole-output equality assertion fails - // on changes that have nothing to do with this test. - assert!( - with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"), - "HIVE_PEER_CA_PATHS must not change the CA list — the store's \ - root inside the hive's own trust bundle is the only anchor \ - embedded:\n{with_stale_peer_env}" - ); - assert!( - !with_stale_peer_env.contains("peer-ca-"), - "no peer-ca-.pem may be embedded, however HIVE_PEER_CA_PATHS \ - is set:\n{with_stale_peer_env}" - ); - assert!( - !stale_peer_env_only.contains("security.pki.certificateFiles"), - "a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \ - reference on its own:\n{stale_peer_env_only}" - ); - assert!( - !without_ca.contains("security.pki.certificateFiles"), - "no certificateFiles reference without any CA signal:\n{without_ca}" + assert_eq!( + out, + " services.hyperhive.agent.bao.servicesPkiMount = \"pki\";\n" ); } diff --git a/hive-forge/Cargo.toml b/hive-forge/Cargo.toml index 5491547a..1b6b6053 100644 --- a/hive-forge/Cargo.toml +++ b/hive-forge/Cargo.toml @@ -26,8 +26,8 @@ reqwest = { workspace = true, features = [ # `rustls` carries the platform verifier, which reads the OS trust store # (and honours `SSL_CERT_FILE`), so hive-forge validates the hive # gateway's self-signed leaf once the store's root is in the agent's trust - # store — embedded into agent flakes via `security.pki.certificateFiles` - # — while public CAs still validate. + # store (an agent fetches it from the secret store at boot and points + # `SSL_CERT_FILE` at it) while public CAs still validate. # # The separate `rustls-native-certs` / `webpki-roots` features this entry # used to name were 0.13.1-era and no longer exist; naming a feature that diff --git a/nix/agent-modules/default.nix b/nix/agent-modules/default.nix index b8949cc5..5db3dbdc 100644 --- a/nix/agent-modules/default.nix +++ b/nix/agent-modules/default.nix @@ -43,6 +43,7 @@ in ./frontend.nix ./github.nix ./github-token.nix + ./hive-ca.nix ./hive-facts.nix ./logs.nix ./matrix.nix diff --git a/nix/agent-modules/hive-ca.nix b/nix/agent-modules/hive-ca.nix new file mode 100644 index 00000000..94effef1 --- /dev/null +++ b/nix/agent-modules/hive-ca.nix @@ -0,0 +1,196 @@ +# The swarm's root certificate in this agent's trust, fetched from the store +# at boot by the agent itself. +# +# The root is the store's services `pki` mount's CA. It does not exist when the +# agent is built, so the meta flake names no CA file: this unit reads it from +# the store with the agent's own certificate (./bao.nix keeps it current) and +# writes two files, both public: +# +# - `bundle`: the system CAs plus the swarm root. `SSL_CERT_FILE` replaces the +# trust store of Rust (rustls platform verifier), Go and OpenSSL clients, so +# it carries both halves. +# - `root`: the swarm root alone, for `NODE_EXTRA_CA_CERTS`, which adds to +# Node's store (Claude Code). +# +# A process reads its trust when it starts, so a rotated root reaches the +# consumers below at their next start. They `Requires=` this unit, which keeps +# one from running with a store that lacks the root. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.services.hyperhive.agent.bao; + agentCfg = config.services.hyperhive.agent; + + agentName = agentCfg.user.name; + + # The server CA id ./bao.nix loads. + serverCaCredential = "hive-agent-bao-server-ca"; + + unitName = "hive-agent-trust-bundle"; + runtimeDir = unitName; + bundleFile = "/run/${runtimeDir}/bundle.pem"; + rootFile = "/run/${runtimeDir}/root.pem"; + + systemBundle = "/etc/ssl/certs/ca-certificates.crt"; + + configured = cfg.addr != null; + + storeRetry = import ../host-modules/lib/store-retry.nix { }; + + # Units that dial a swarm service over TLS. The first group is defined + # whenever the agent module is; the rest exist under the same condition as + # their definitions, because ordering on a unit that is not defined would + # generate an inert one. + consumers = [ + "hive-agent" + "hive-bash-daemon" + "hive-subagent-daemon" + "hive-mcp-http" + "hive-forge-notify" + ] + ++ lib.optional (agentCfg.matrixAccounts != { }) "hive-matrix-daemon" + ++ lib.optional (agentCfg.github.enable) "hive-github-notify" + ++ lib.optional (agentCfg.icon != null && agentCfg.forge.url != null) "forge-avatar-sync"; + + trustEnv = { + SSL_CERT_FILE = bundleFile; + NODE_EXTRA_CA_CERTS = rootFile; + }; +in +{ + options.services.hyperhive.agent.bao.servicesPkiMount = lib.mkOption { + type = lib.types.str; + default = "pki"; + example = "pki"; + description = '' + Mount path of the store's services `pki` engine, whose root is the one + anchor every swarm service certificate chains to. Set by the generated + meta flake from the host's + `services.hyperhive.deploy.bao.servicesPkiMountPath`. + `${unitName}.service` reads `/ca/pem` from it. + ''; + }; + + config = lib.mkIf configured { + # Every process, including a bash task's children, resolves trust through + # these, as it resolves `HIVE_FORGE_TOKEN_FILE`. + systemd.globalEnvironment = trustEnv; + environment.variables = trustEnv; + + systemd.services = { + ${unitName} = { + description = "fetch the swarm root certificate from the secret store"; + after = [ + "network.target" + "hive-agent-bao-identity.service" + ]; + wantedBy = [ "multi-user.target" ]; + path = [ + pkgs.coreutils + pkgs.curl + pkgs.openssl + ]; + inherit (storeRetry) startLimitBurst startLimitIntervalSec; + serviceConfig = storeRetry.serviceConfig // { + Type = "oneshot"; + RemainAfterExit = false; + TimeoutStartSec = 60; + User = agentName; + Group = agentName; + RuntimeDirectory = runtimeDir; + # Public certificates, read by every consumer's user. + RuntimeDirectoryMode = "0755"; + RuntimeDirectoryPreserve = "yes"; + UMask = "0022"; + LoadCredential = [ serverCaCredential ]; + }; + # Overrides the global value: this unit runs before the bundle exists, + # and its own TLS is checked against the credential below or the + # system bundle. + environment = { + SSL_CERT_FILE = lib.mkForce systemBundle; + BAO_ADDR = cfg.addr; + BAO_CLIENT_CERT = cfg.clientCertFile; + BAO_CLIENT_KEY = cfg.clientKeyFile; + }; + script = '' + set -euo pipefail + + for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do + if [ ! -s "$f" ]; then + echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2 + exit 1 + fi + done + + cacert= + if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then + cacert="$CREDENTIALS_DIRECTORY/${serverCaCredential}" + echo "trusting bao via the delivered server CA credential" >&2 + else + echo "no bao server CA credential delivered; trusting the system bundle" >&2 + fi + + dir=/run/${runtimeDir} + err=$(mktemp -p "$dir" err.XXXXXX) + raw=$(mktemp -p "$dir" raw.XXXXXX) + root=$(mktemp -p "$dir" root.XXXXXX) + bundle=$(mktemp -p "$dir" bundle.XXXXXX) + trap 'rm -f "$err" "$raw" "$root" "$bundle"' EXIT + + # The listener demands a client certificate and the endpoint takes no + # token, so no login precedes this. + if ! curl --fail --silent --show-error \ + --connect-timeout 10 --max-time 30 \ + ''${cacert:+--cacert "$cacert"} \ + --cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \ + --output "$raw" \ + ${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then + echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2 + cat "$err" >&2 + exit 1 + fi + + # Re-encoded rather than copied: it fails on a body that is not a + # certificate and ends the file in the newline the concatenation + # below relies on. + if ! openssl x509 -in "$raw" -out "$root" 2>"$err"; then + echo "the store's ${cfg.servicesPkiMount} root does not parse as a certificate." >&2 + cat "$err" >&2 + exit 1 + fi + + cat ${systemBundle} "$root" >"$bundle" + + # Neither file moves unless it changed, so a reader never sees a half + # write and the timer's rerun is silent. + if cmp -s "$bundle" ${bundleFile} && cmp -s "$root" ${rootFile}; then + echo "the store's ${cfg.servicesPkiMount} root is unchanged" + exit 0 + fi + chmod 0644 "$root" "$bundle" + mv -f "$root" ${rootFile} + mv -f "$bundle" ${bundleFile} + echo "wrote the swarm root certificate from the store's ${cfg.servicesPkiMount} mount" + ''; + }; + } + // lib.genAttrs consumers (_: { + after = [ "${unitName}.service" ]; + requires = [ "${unitName}.service" ]; + }); + + systemd.timers.${unitName} = { + description = "re-read the swarm root certificate from the secret store"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnUnitInactiveSec = "1h"; + RandomizedDelaySec = "1min"; + }; + }; + }; +} diff --git a/nix/host-modules/glue-hive-tls-controllers.nix b/nix/host-modules/glue-hive-tls-controllers.nix index 07c9041f..938095be 100644 --- a/nix/host-modules/glue-hive-tls-controllers.nix +++ b/nix/host-modules/glue-hive-tls-controllers.nix @@ -2,13 +2,10 @@ # # ONE PAIRING PER FILE — hive-c0re and the swarm controller ← the trust bundle # ./hive-tls.nix assembles. Deleting this leaves daemons that trust only the -# platform roots: hive-c0re's lifecycle cannot hand agents the bundle, and the -# swarm-queue clients die at `invalid peer certificate: UnknownIssuer` against -# an authelia signed by the swarm CA. +# platform roots: the swarm-queue clients die at +# `invalid peer certificate: UnknownIssuer` against an authelia signed by the +# swarm CA. # -# `HIVE_TLS_CA_PATH` signals the hive-c0re lifecycle that a trust bundle exists: -# it bind-mounts the file (read-only, public certs ONLY — never a key) into each -# agent container, and the meta flake wires the per-agent trust bundle. # `*_OIDC_CA_FILE` names the same anchor for the swarm-queue clients that mint a # token from authelia over TLS. `swarm-queue-client` builds a bare # `reqwest::Client`, so it trusts only the platform roots without it. Where the @@ -43,7 +40,6 @@ in lib.mkMerge [ (lib.mkIf deployCfg.hive-controller.enable { systemd.services.hive-c0re.environment = { - HIVE_TLS_CA_PATH = trustBundle; HIVE_C0RE_OIDC_CA_FILE = trustBundle; }; }) diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index aff9c44c..049553c7 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -278,6 +278,9 @@ in # environment wholesale would inherit the hive's certificate paths with it, # and those name a credential no agent may present. HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}"; + # The mount whose root an agent fetches to trust the swarm's services; the + # same value ../hive-tls.nix reads the root from. + HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = baoDeploy.servicesPkiMountPath; # `%d`, not the paths themselves: the key is `0600` root-owned and this # daemon runs as hive-core, so it never gets read access to the original. # See the LoadCredential in ./default.nix. diff --git a/nix/module-eval/agent-forge-bao.nix b/nix/module-eval/agent-forge-bao.nix index 8f4ccb80..8f51ebc8 100644 --- a/nix/module-eval/agent-forge-bao.nix +++ b/nix/module-eval/agent-forge-bao.nix @@ -47,6 +47,38 @@ let in let cases = [ + { + # ../agent-modules/hive-ca.nix. Presence arm first (the unit, the + # timer, the trust variables, and every consumer ordered after the + # unit and requiring it), then the absence arm for an agent with no + # store, which has no root to fetch. + name = "an agent with a store address fetches the swarm root and its consumers wait for it"; + ok = + let + svc = agentForgeBao.systemd.services; + unit = "hive-agent-trust-bundle.service"; + bundle = "/run/hive-agent-trust-bundle/bundle.pem"; + consumers = [ + "hive-agent" + "hive-bash-daemon" + "hive-subagent-daemon" + "hive-mcp-http" + "hive-forge-notify" + "forge-avatar-sync" + ]; + in + svc ? hive-agent-trust-bundle + && agentForgeBao.systemd.timers ? hive-agent-trust-bundle + && lib.hasInfix "/v1/pki/ca/pem" svc.hive-agent-trust-bundle.script + && agentForgeBao.systemd.globalEnvironment.SSL_CERT_FILE == bundle + && + agentForgeBao.systemd.globalEnvironment.NODE_EXTRA_CA_CERTS + == "/run/hive-agent-trust-bundle/root.pem" + && lib.all (c: lib.elem unit svc.${c}.after && lib.elem unit svc.${c}.requires) consumers + && !(agentForgeNoBao.systemd.services ? hive-agent-trust-bundle) + && !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE) + && !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ])); + } { # The whole switch is the store address, as for the queue credential. name = "an agent with a store address fetches its forge token"; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 5e8d1db9..e6060888 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -317,6 +317,12 @@ let in e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR; } + { + name = "an agent is told the mount its swarm root is read from, only where it is told the store"; + ok = + allLocal.systemd.services.hive-c0re.environment.HIVE_AGENT_BAO_SERVICES_PKI_MOUNT == "pki" + && !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_SERVICES_PKI_MOUNT); + } { # A hive with no certificate of its own can collect no agent's identity, # so forwarding an address would name a store nothing in the container diff --git a/nix/module-eval/hive-tls.nix b/nix/module-eval/hive-tls.nix index abab4d62..51ba656f 100644 --- a/nix/module-eval/hive-tls.nix +++ b/nix/module-eval/hive-tls.nix @@ -693,10 +693,7 @@ let let bundle = "/var/lib/hive-tls/trust-bundle.pem"; envOf = m: unit: m.systemd.services.${unit}.environment or { }; - hiveVars = [ - "HIVE_TLS_CA_PATH" - "HIVE_C0RE_OIDC_CA_FILE" - ]; + hiveVars = [ "HIVE_C0RE_OIDC_CA_FILE" ]; anyOf = m: unit: vars: lib.any (v: (envOf m unit) ? ${v}) vars;