vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

agents fetch the hive CA from bao at boot instead of the meta flake embedding it

Browse source
This commit is contained in:
atlas 2026-10-19 18:46:34 +02:00 • committed by mara
commit d547b521f6
12 changed files with 329 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

303
hive-c0re/src/meta.rs
View file

@ -100,15 +100,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default();
let initial = !dir.join(".git").exists();
// Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in
// lockstep with its host source so the build-time `certificateFiles`
// list render_flake emits always resolves. Empty when the hive has no
// trust bundle.
let (ca_files, ca_changed) = ca_embed_state(&dir);
// Skip only when both the flake AND the embedded CA are unchanged — a
// CA rotation with an otherwise-identical flake must still re-commit.
if !initial && on_disk == new_flake && !ca_changed {
if !initial && on_disk == new_flake {
return Ok(());
}
@ -130,10 +122,9 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
std::fs::write(&flake_path, &new_flake)
.with_context(|| format!("write {}", flake_path.display()))?;
// Materialise the embedded CA list next to flake.nix + drop any stale
// CA file; `ca_touched` is every filename written or removed, staged
// for commit below. Public CA certs only; no private key is embedded.
let ca_touched = materialise_ca_files(&dir, &ca_files)?;
// Agents fetch the swarm root from the store at boot, so no CA file sits
// next to the flake; drop the ones earlier renders left behind.
let ca_touched = remove_stale_ca_files(&dir);
// Reconcile topology.json against the live agent set — adds
// newly-spawned agents and drops removed ones. Idempotent; when
@ -175,8 +166,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
// contain '/flake.nix'". Lock then commit once with both
// flake.nix and flake.lock — single commit per change.
git(&dir, &["add", "flake.nix"]).await?;
// Stage every embedded CA file we wrote or removed (trust bundle +
// stale peer CAs). `git add <path>` stages a deletion when the path is
// Stage every stale CA file removed above. `git add <path>` stages a deletion when the path is
// tracked and now gone; best-effort so the never-tracked-and-absent case
// (pathspec mismatch) is a harmless no-op.
for name in &ca_touched {
@ -204,9 +194,8 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
.filter_map(|f| match f.as_str() {
"flake.nix" => Some("flake"),
"flake.lock" => Some("lock"),
"hive-ca.pem" => Some("hive-ca"),
// Only ever a REMOVAL now; see `is_embedded_ca_name`.
f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"),
// Only ever a removal; see `remove_stale_ca_files`.
f if is_stale_ca_name(f) => Some("stale-ca"),
"topology.json" => Some("topology"),
_ => None,
})
@ -394,6 +383,9 @@ const FORWARDED_VARS: &[&str] = &[
// hive's store environment wholesale — the address is the only part of it
// an agent may have, the certificate beside it being the hive's own.
"HIVE_AGENT_BAO_ADDR",
// The mount path of the store's services `pki` engine, whose root an agent
// fetches to trust the swarm. Forwarded with the address, under the same gate.
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT",
// Where an agent's `swarm-logs` queries, computed by the host for the
// same reason as the two above: it is a gateway address, and a container
// cannot derive it from anything it holds.
@ -437,6 +429,12 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[
// environment, so an agent whose option says no store and whose env says
// otherwise would have a check that is simply not there.
("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"),
// Read at build time to bake the root-fetch URL into
// `nix/agent-modules/hive-ca.nix`'s unit.
(
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT",
"services.hyperhive.agent.bao.servicesPkiMount",
),
// Read at build time to decide whether `swarm-logs` is installed at all —
// see `nix/agent-modules/logs.nix`. Both halves wired for this map's own
// reason: an agent whose option says no log store and whose env says
@ -537,30 +535,6 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> {
.collect()
}
/// Filename the hive's own trust anchors are embedded under at the
/// meta-flake root — the only entry of the embedded-CA list
/// `render_flake` references in `security.pki.certificateFiles` (see
/// `embedded_ca_files`).
const HIVE_CA_FILE: &str = "hive-ca.pem";
/// Host path of the hive's TLS trust anchors, when self-signed TLS is
/// active. The hyperhive TLS module sets `HIVE_TLS_CA_PATH` in hive-c0re's env
/// (to `<deploy.hive-controller.tls.stateDir>/trust-bundle.pem`) whenever the
/// gateway serves a leaf issued by the secret store's `pki` mount. The
/// file holds that mount's root, the one anchor every hive's gateway leaf
/// chains to, and is copied verbatim, which `security.pki.certificateFiles`
/// accepts; nothing here parses it. Returns `Some(path)` only when the var
/// is set AND the cert exists on disk — so render + write stay consistent
/// (we never emit a `certificateFiles` reference to a file we didn't
/// embed). Only the public cert is ever read here.
fn hive_ca_source() -> Option<String> {
let path = std::env::var("HIVE_TLS_CA_PATH").ok()?;
if path.is_empty() || !std::path::Path::new(&path).is_file() {
return None;
}
Some(path)
}
/// Extra `OTEL_RESOURCE_ATTRIBUTES` entries for every agent, read off
/// hive-c0re's own unit env
/// (`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`).
@ -574,105 +548,32 @@ fn otel_extra_resource_attributes() -> Option<String> {
.filter(|v| !v.is_empty())
}
/// The ordered set of CA certs embedded next to the meta flake, as
/// `(filename, host_source_path)` — now just the hive's own trust
/// anchors as `hive-ca.pem`, when self-signed TLS is active.
///
/// That single file carries the store's `pki` root (the hyperhive TLS
/// module writes it into the trust bundle), so every hive's gateway leaf
/// validates from it and nothing is embedded per peer.
///
/// `render_flake` emits exactly these filenames into
/// `security.pki.certificateFiles` and `sync_agents` materialises them,
/// so the rendered reference and the embedded files always agree.
fn embedded_ca_files() -> Vec<(String, String)> {
let mut out = Vec::new();
if let Some(p) = hive_ca_source() {
out.push((HIVE_CA_FILE.to_owned(), p));
}
out
}
/// Write each desired embedded CA file next to `flake.nix` and remove
/// any stale one (self-signed TLS turned off, or a leftover peer CA),
/// so the flake never references a file we didn't write. Returns every
/// filename written or removed, for the caller to stage. The public CA
/// certs only; no private key is ever embedded.
fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result<Vec<String>> {
let desired: std::collections::HashSet<&str> =
ca_files.iter().map(|(n, _)| n.as_str()).collect();
let mut touched: Vec<String> = Vec::new();
/// Remove the CA files earlier renders embedded next to `flake.nix`
/// (`hive-ca.pem`, and the per-peer `peer-ca-<N>.pem` before it), returning
/// every filename removed so the caller stages the deletion. Nothing writes
/// one now: agents fetch the swarm root from the store at boot, so a file left
/// here is referenced by nothing.
fn remove_stale_ca_files(dir: &Path) -> Vec<String> {
let mut removed = Vec::new();
if let Ok(entries) = std::fs::read_dir(dir) {
for e in entries.flatten() {
let fname = e.file_name();
let Some(name) = fname.to_str() else { continue };
if is_embedded_ca_name(name) && !desired.contains(name) {
let _ = std::fs::remove_file(dir.join(name));
touched.push(name.to_owned());
if is_stale_ca_name(name) && std::fs::remove_file(dir.join(name)).is_ok() {
removed.push(name.to_owned());
}
}
}
for (name, content) in ca_files {
let path = dir.join(name);
std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?;
touched.push(name.clone());
}
Ok(touched)
removed
}
/// True for a filename the embedded-CA machinery owns. Lets
/// `sync_agents` find stale CA files to clean up (a CA dropped from
/// config) without touching unrelated meta files.
///
/// ⚠️ Still matches `peer-ca-<N>.pem`, which `embedded_ca_files` no
/// longer produces — deliberately. Cleanup is driven by
/// "recognised but not desired", so this arm is exactly what removes a
/// per-peer CA file still sitting at a meta root. Drop it and those files
/// are orphaned at every meta root forever, referenced by nothing and
/// cleaned by nobody.
fn is_embedded_ca_name(name: &str) -> bool {
name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name))
}
/// True when `name` ends in a `.pem` extension (case-insensitive). Split
/// out so the embedded-CA filename checks share one spelling and dodge
/// clippy's case-sensitive-extension lint.
fn has_pem_ext(name: &str) -> bool {
Path::new(name)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("pem"))
}
/// Embedded-CA state for the meta repo: `(desired_files, changed)`.
/// `desired_files` is `(filename, contents)` for every CA that should sit
/// next to flake.nix. `changed` is true when the on-disk set differs in
/// any way — the file's contents changed, a CA appeared, or a
/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is
/// no longer wanted (stale, to be removed). Drives both the re-commit
/// decision and the materialise/cleanup in `sync_agents`, so a CA
/// rotation re-commits even when the flake itself is byte-identical.
fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) {
let desired: Vec<(String, String)> = embedded_ca_files()
.into_iter()
.filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c)))
.collect();
let desired_names: std::collections::HashSet<&str> =
desired.iter().map(|(n, _)| n.as_str()).collect();
let mut changed = desired.iter().any(|(name, content)| {
std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content
});
// A previously-embedded CA file no longer wanted → stale (removal is
// a change even when every desired file already matches on disk).
if !changed && let Ok(entries) = std::fs::read_dir(dir) {
changed = entries.flatten().any(|e| {
e.file_name()
.to_str()
.is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name))
});
}
(desired, changed)
/// True for the filenames [`remove_stale_ca_files`] owns.
fn is_stale_ca_name(name: &str) -> bool {
name == "hive-ca.pem"
|| (name.starts_with("peer-ca-")
&& Path::new(name)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("pem")))
}
/// Resolve every custom agent's [`AgentPin`] and return `agents` with it set.
@ -862,26 +763,6 @@ fn render_flake(
" services.hyperhive.agent.claudeCodePath = \"{path}\";"
);
}
// CA trust: embed the hive's trust bundle so each agent validates
// the store's `pki` root at build time (one file; see
// `hive_ca_source`). `certificateFiles` is build-time, so the certs
// travel with the flake source: `sync_agents` writes `./hive-ca.pem`
// next to flake.nix and stages it. Only public CA certs are embedded;
// no private key ever leaves the host. The filename list matches
// `sync_agents` exactly (both derive it from `embedded_ca_files`), so
// we never reference a file we didn't embed; emitted only when the
// list is non-empty.
let ca_refs: Vec<String> = embedded_ca_files()
.into_iter()
.map(|(name, _)| format!("./{name}"))
.collect();
if !ca_refs.is_empty() {
let _ = writeln!(
out,
" security.pki.certificateFiles = [ {} ];",
ca_refs.join(" ")
);
}
// Extra OTEL resource labels (`deploy.hive-controller.otel.*`) are the one
// OTEL value rendered here; enablement and the collector endpoint are
// agent-module defaults.
@ -2005,24 +1886,13 @@ mod tests {
}
#[test]
fn render_flake_embeds_hive_ca_when_signalled() {
// When the hyperhive TLS module signals a trust bundle via
// HIVE_TLS_CA_PATH (and the cert exists), the agent module must
// trust it at build time via security.pki.certificateFiles. Absent
// the signal, no reference is emitted (so the flake doesn't point at
// a file that was never embedded).
fn render_flake_embeds_no_ca() {
// Agents fetch the swarm root from the store at boot; the flake names
// no CA file whatever the host env says, including the variable
// hive-c0re used to embed from.
//
// Serialised against every other env-mutating test in the crate. Taken
// before the fixture files so the guard covers the whole window in
// which HIVE_TLS_CA_PATH / HIVE_PEER_CA_PATHS are perturbed.
// Serialised against every other env-mutating test in the crate.
let _env = crate::test_env::lock();
let ca_file = std::env::temp_dir().join(format!("hive-ca-test-{}.pem", std::process::id()));
std::fs::write(
&ca_file,
"-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n",
)
.expect("write temp CA");
let render = || {
render(
"git+https://forge.example/hyperhive/hyperhive.git?rev=0123abcd",
@ -2031,70 +1901,59 @@ mod tests {
&[sample_spec("alice", 9001)],
)
};
// A leftover peer-CA file + the env var that names it. Both must be
// inert: the store's root rides in the hive's own trust bundle, so
// nothing per-peer is embedded.
let stale_peer =
std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id()));
std::fs::write(
&stale_peer,
"-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n",
)
.expect("write stale peer CA");
// SAFETY: serialised by the guard at the top of this test; restored
// before returning.
unsafe {
std::env::remove_var("HIVE_PEER_CA_PATHS");
std::env::set_var("HIVE_TLS_CA_PATH", &ca_file);
std::env::set_var("HIVE_TLS_CA_PATH", "/nonexistent/trust-bundle.pem");
}
let with_ca = render();
unsafe {
std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string());
}
let with_stale_peer_env = render();
// The stale var alone, with no trust bundle: must produce nothing.
let flake = render();
unsafe {
std::env::remove_var("HIVE_TLS_CA_PATH");
}
let stale_peer_env_only = render();
unsafe {
std::env::remove_var("HIVE_PEER_CA_PATHS");
// Presence control: the render does carry per-agent module config, so
// the absences below are not an empty render.
assert!(
flake.contains("services.hyperhive.agent."),
"per-agent module block missing:\n{flake}"
);
for needle in ["hive-ca.pem", "certificateFiles", "peer-ca-"] {
assert!(
!flake.contains(needle),
"the flake must not name {needle}:\n{flake}"
);
}
let without_ca = render();
let _ = std::fs::remove_file(&ca_file);
let _ = std::fs::remove_file(&stale_peer);
}
assert!(
with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
"CA cert must be wired into certificateFiles when signalled:\n{with_ca}"
#[test]
fn stale_ca_files_are_removed_and_nothing_else() {
let dir = tempfile::tempdir().expect("tempdir");
for name in ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM", "flake.nix"] {
std::fs::write(dir.path().join(name), "x").expect("write");
}
// Not CA files this machinery owns, though they look similar.
for name in ["peer-ca-0.txt", "other-ca.pem"] {
std::fs::write(dir.path().join(name), "x").expect("write");
}
let mut removed = remove_stale_ca_files(dir.path());
removed.sort();
assert_eq!(removed, ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM"]);
for kept in ["flake.nix", "peer-ca-0.txt", "other-ca.pem"] {
assert!(dir.path().join(kept).exists(), "{kept} must survive");
}
assert!(remove_stale_ca_files(dir.path()).is_empty());
}
#[test]
fn services_pki_mount_is_forwarded_as_both_env_and_option() {
assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"));
let mut out = String::new();
push_forwarded_var_options(
&mut out,
&[("HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", "pki".to_string())],
);
// The regression guard, scoped to the CA list rather than the whole
// render. Comparing the two flakes wholesale looks stronger and is
// actually FLAKY: `cargo test` runs these in parallel threads and
// sibling tests mutate process env (OTEL, forge URLs) between the
// two `render()` calls, so a whole-output equality assertion fails
// on changes that have nothing to do with this test.
assert!(
with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
"HIVE_PEER_CA_PATHS must not change the CA list — the store's \
root inside the hive's own trust bundle is the only anchor \
embedded:\n{with_stale_peer_env}"
);
assert!(
!with_stale_peer_env.contains("peer-ca-"),
"no peer-ca-<N>.pem may be embedded, however HIVE_PEER_CA_PATHS \
is set:\n{with_stale_peer_env}"
);
assert!(
!stale_peer_env_only.contains("security.pki.certificateFiles"),
"a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \
reference on its own:\n{stale_peer_env_only}"
);
assert!(
!without_ca.contains("security.pki.certificateFiles"),
"no certificateFiles reference without any CA signal:\n{without_ca}"
assert_eq!(
out,
" services.hyperhive.agent.bao.servicesPkiMount = \"pki\";\n"
);
}