refactor(nix): swarm.peers becomes swarm.hives, a directory of every hive
One attrset describing every hive in the swarm including this one,
identical on every host, with hiveName selecting which entry is us.
"My peers" is derived (swarm.peerHives) rather than declared.
Every field in the old per-host peer list was intrinsic to the hive it
described, never to the pair -- so the list was a directory each host
kept its own copy of. Beyond the deduplication it removes a bug class:
two hosts could hold different endpoints for the same third hive with
nothing to detect the disagreement.
Drops the per-hive caCert. Trust inside a swarm derives from the swarm
root, which every hive chains to. What that genuinely removes is
trusting a hive whose root this swarm does not own -- a cross-swarm
problem that wants a mechanism of its own, not a field that happened to
work.
The matrix container's certificateFiles block goes with it and could
NOT be migrated: that list is read at build time and the swarm root is
a runtime file (its key must never enter the store), so there is no
build-time name to put there. caCert being a nix path was precisely
what made it the build-time distribution channel. Agents are unaffected
-- hive-tls folds the root into the hive trust bundle and the meta
renderer embeds that one file. Tracked separately.
Migration is an assertion plus warnings, not a rename: hives is peers
union {self}, and the set gains a member no existing config has written
down. A rename migrates a name and a default can re-root a meaning;
neither can conjure a new member. The warning explains, the self-entry
assertion stops the build.
This commit is contained in:
atlas
parent
048bdd29a8
commit
433b294099
19 changed files with 484 additions and 293 deletions
|
|
@ -122,7 +122,8 @@ pub(super) struct StateSnapshot {
|
|||
swarm_name: Option<String>,
|
||||
/// Peer hives in the same swarm. Parsed from `HYPERHIVE_PEERS`
|
||||
/// (JSON array of `{domain,cert_fingerprint}` objects, emitted by
|
||||
/// the c0re NixOS module from `services.hyperhive.swarm.peers`).
|
||||
/// the c0re NixOS module from `services.hyperhive.swarm.peerHives`
|
||||
/// — the swarm's `hives` directory minus this hive).
|
||||
/// Empty on single-hive deploys. Feeds the P33RS dashboard tab.
|
||||
peer_hives: Vec<PeerHiveView>,
|
||||
/// Server-level warnings for the dashboard's top-of-page banner
|
||||
|
|
@ -162,9 +163,9 @@ async fn infra_container_views() -> Vec<InfraContainerView> {
|
|||
|
||||
/// One peer hive for the P33RS dashboard tab. Derived from
|
||||
/// `HYPERHIVE_PEERS` env; `url` is the peer's HTTPS dashboard root.
|
||||
/// `cert_fingerprint` is `Some("sha256:<hex64>")` when the peer uses a
|
||||
/// self-signed cert and the operator pinned its fingerprint in
|
||||
/// `services.hyperhive.swarm.peers`.
|
||||
/// `cert_fingerprint` is `Some("sha256:<hex64>")` when the operator
|
||||
/// pinned the peer's leaf in `services.hyperhive.swarm.hives`, which a
|
||||
/// hive under the swarm root CA does not need.
|
||||
#[derive(Serialize)]
|
||||
struct PeerHiveView {
|
||||
name: String,
|
||||
|
|
@ -450,7 +451,7 @@ pub(super) async fn api_state(
|
|||
/// Parse `HYPERHIVE_PEERS` env var into dashboard-ready `PeerHiveView`
|
||||
/// entries. The env var is a JSON array of `{domain, cert_fingerprint}`
|
||||
/// objects emitted by the c0re NixOS module from
|
||||
/// `services.hyperhive.swarm.peers`. Each entry becomes
|
||||
/// `services.hyperhive.swarm.peerHives`. Each entry becomes
|
||||
/// `{ name: domain, url: "https://domain/" }` for the P33RS tab.
|
||||
/// Returns empty vec when unset (single-hive deploy).
|
||||
fn parse_peer_hives() -> Vec<PeerHiveView> {
|
||||
|
|
|
|||
|
|
@ -108,11 +108,10 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default();
|
||||
let initial = !dir.join(".git").exists();
|
||||
|
||||
// Embedded-CA list (self-signed hive CA + peer CAs): keep the
|
||||
// `./hive-ca.pem` / `./peer-ca-<N>.pem` files at the meta root in
|
||||
// lockstep with their host sources so the build-time `certificateFiles`
|
||||
// list render_flake emits always resolves. Empty when neither a
|
||||
// self-signed hive CA nor any peer CA is configured.
|
||||
// Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in
|
||||
// lockstep with its host source so the build-time `certificateFiles`
|
||||
// list render_flake emits always resolves. Empty when no self-signed
|
||||
// hive CA is configured.
|
||||
let (ca_files, ca_changed) = ca_embed_state(&dir);
|
||||
|
||||
// Skip only when both the flake AND the embedded CA are unchanged — a
|
||||
|
|
@ -217,6 +216,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
"flake.nix" => Some("flake"),
|
||||
"flake.lock" => Some("lock"),
|
||||
"hive-ca.pem" => Some("hive-ca"),
|
||||
// Only ever a REMOVAL now; see `is_embedded_ca_name`.
|
||||
f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"),
|
||||
"topology.json" => Some("topology"),
|
||||
"capabilities.json" => Some("capabilities"),
|
||||
|
|
@ -809,10 +809,10 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Filename the hive's own self-signed CA cert is embedded under at the
|
||||
/// meta-flake root. One entry of the embedded-CA list `render_flake`
|
||||
/// references in `security.pki.certificateFiles` (see `embedded_ca_files`);
|
||||
/// peer CAs sit alongside it as `peer-ca-<N>.pem`.
|
||||
/// Filename the hive's own trust anchors are embedded under at the
|
||||
/// meta-flake root — the only entry of the embedded-CA list
|
||||
/// `render_flake` references in `security.pki.certificateFiles` (see
|
||||
/// `embedded_ca_files`).
|
||||
const HIVE_CA_FILE: &str = "hive-ca.pem";
|
||||
|
||||
/// Host path of the hive's TLS trust anchors, when self-signed TLS is
|
||||
|
|
@ -833,23 +833,6 @@ fn hive_ca_source() -> Option<String> {
|
|||
Some(path)
|
||||
}
|
||||
|
||||
/// Host paths of peer-hive root CA certificates, from `HIVE_PEER_CA_PATHS`
|
||||
/// (colon-separated; set by hive-c0re.nix from `swarm.peers.<d>.caCert`).
|
||||
/// Each is embedded alongside the hive CA so a peer's CA is trusted
|
||||
/// everywhere the hive's own internal CA is — i.e. by every agent. Empty
|
||||
/// segments and paths that don't resolve to a file are dropped, so we
|
||||
/// never reference a `certificateFiles` entry we couldn't embed.
|
||||
fn peer_ca_sources() -> Vec<String> {
|
||||
let Ok(raw) = std::env::var("HIVE_PEER_CA_PATHS") else {
|
||||
return Vec::new();
|
||||
};
|
||||
raw.split(':')
|
||||
.map(str::trim)
|
||||
.filter(|p| !p.is_empty() && std::path::Path::new(p).is_file())
|
||||
.map(ToOwned::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Hive-wide OTEL config injected into every agent's build, read off
|
||||
/// hive-c0re's own unit env (set from `services.hyperhive.otel.*` in
|
||||
/// `nix/modules/hive-c0re.nix`). A present, non-empty
|
||||
|
|
@ -906,9 +889,16 @@ fn otel_config() -> Option<OtelConfig> {
|
|||
}
|
||||
|
||||
/// The ordered set of CA certs embedded next to the meta flake, as
|
||||
/// `(filename, host_source_path)`. The self-signed hive CA (when active)
|
||||
/// is `hive-ca.pem`; each peer CA is `peer-ca-<N>.pem` in declaration
|
||||
/// order. `render_flake` emits exactly these filenames into
|
||||
/// `(filename, host_source_path)` — now just the hive's own trust
|
||||
/// anchors as `hive-ca.pem`, when self-signed TLS is active.
|
||||
///
|
||||
/// That single file already carries the swarm root (`hive-tls.nix`
|
||||
/// writes the hive CA *and* the root it is issued under into the trust
|
||||
/// bundle), so every hive under the swarm root validates from it. Which
|
||||
/// is why the per-peer CAs this used to append are gone: they said the
|
||||
/// same thing once per peer.
|
||||
///
|
||||
/// `render_flake` emits exactly these filenames into
|
||||
/// `security.pki.certificateFiles` and `sync_agents` materialises them,
|
||||
/// so the rendered reference and the embedded files always agree.
|
||||
fn embedded_ca_files() -> Vec<(String, String)> {
|
||||
|
|
@ -916,9 +906,6 @@ fn embedded_ca_files() -> Vec<(String, String)> {
|
|||
if let Some(p) = hive_ca_source() {
|
||||
out.push((HIVE_CA_FILE.to_owned(), p));
|
||||
}
|
||||
for (i, p) in peer_ca_sources().into_iter().enumerate() {
|
||||
out.push((format!("peer-ca-{i}.pem"), p));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
|
|
@ -949,9 +936,16 @@ fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result<Vec
|
|||
Ok(touched)
|
||||
}
|
||||
|
||||
/// True for a filename `embedded_ca_files` can produce — the hive CA or
|
||||
/// a `peer-ca-<N>.pem`. Lets `sync_agents` find stale CA files to clean
|
||||
/// up (a CA dropped from config) without touching unrelated meta files.
|
||||
/// True for a filename the embedded-CA machinery owns. Lets
|
||||
/// `sync_agents` find stale CA files to clean up (a CA dropped from
|
||||
/// config) without touching unrelated meta files.
|
||||
///
|
||||
/// ⚠️ Still matches `peer-ca-<N>.pem`, which `embedded_ca_files` no
|
||||
/// longer produces — deliberately. Cleanup is driven by
|
||||
/// "recognised but not desired", so this arm is exactly what removes the
|
||||
/// peer CAs a hive embedded before per-peer pinning was replaced by the
|
||||
/// swarm root. Drop it and those files are orphaned at every meta root
|
||||
/// forever, referenced by nothing and cleaned by nobody.
|
||||
fn is_embedded_ca_name(name: &str) -> bool {
|
||||
name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name))
|
||||
}
|
||||
|
|
@ -967,13 +961,12 @@ fn has_pem_ext(name: &str) -> bool {
|
|||
|
||||
/// Embedded-CA state for the meta repo: `(desired_files, changed)`.
|
||||
/// `desired_files` is `(filename, contents)` for every CA that should sit
|
||||
/// next to flake.nix (the hive CA + each peer CA). `changed` is true when
|
||||
/// the on-disk set differs in any way — a file's contents changed, a new
|
||||
/// CA appeared, or a previously-embedded CA (`hive-ca.pem` /
|
||||
/// `peer-ca-*.pem`) is no longer wanted (stale, to be removed). Drives
|
||||
/// both the re-commit decision and the materialise/cleanup in
|
||||
/// `sync_agents`, so a CA rotation or a peer-set change re-commits even
|
||||
/// when the flake itself is byte-identical.
|
||||
/// next to flake.nix. `changed` is true when the on-disk set differs in
|
||||
/// any way — the file's contents changed, a CA appeared, or a
|
||||
/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is
|
||||
/// no longer wanted (stale, to be removed). Drives both the re-commit
|
||||
/// decision and the materialise/cleanup in `sync_agents`, so a CA
|
||||
/// rotation re-commits even when the flake itself is byte-identical.
|
||||
fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) {
|
||||
let desired: Vec<(String, String)> = embedded_ca_files()
|
||||
.into_iter()
|
||||
|
|
@ -1170,16 +1163,16 @@ where
|
|||
if let Some(path) = claude_code_path {
|
||||
let _ = writeln!(out, " hyperhive.claudeCodePath = \"{path}\";");
|
||||
}
|
||||
// CA trust: embed every hive-trusted CA so each agent validates them at
|
||||
// build time. The list is the hive's own self-signed CA (when active)
|
||||
// plus every peer-hive root CA (`swarm.peers.<d>.caCert`) — a peer CA is
|
||||
// trusted everywhere the hive's own internal CA is. `certificateFiles` is
|
||||
// build-time, so the certs travel with the flake source: `sync_agents`
|
||||
// writes `./hive-ca.pem` + `./peer-ca-<N>.pem` next to flake.nix and
|
||||
// stages them. Only public CA certs are embedded; no private key ever
|
||||
// leaves the host. The filename list matches `sync_agents` exactly (both
|
||||
// derive it from `embedded_ca_files`), so we never reference a file we
|
||||
// didn't embed; emitted only when the list is non-empty.
|
||||
// CA trust: embed the hive's trust anchors so each agent validates
|
||||
// them at build time — the hive's own self-signed CA when active,
|
||||
// together with the swarm root it is issued under (one file; see
|
||||
// `hive_ca_source`). `certificateFiles` is build-time, so the certs
|
||||
// travel with the flake source: `sync_agents` writes `./hive-ca.pem`
|
||||
// next to flake.nix and stages it. Only public CA certs are embedded;
|
||||
// no private key ever leaves the host. The filename list matches
|
||||
// `sync_agents` exactly (both derive it from `embedded_ca_files`), so
|
||||
// we never reference a file we didn't embed; emitted only when the
|
||||
// list is non-empty.
|
||||
let ca_refs: Vec<String> = embedded_ca_files()
|
||||
.into_iter()
|
||||
.map(|(name, _)| format!("./{name}"))
|
||||
|
|
@ -2169,20 +2162,16 @@ mod tests {
|
|||
)
|
||||
};
|
||||
|
||||
// Two peer-hive CA temp files for the list cases.
|
||||
let peer0 = std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id()));
|
||||
let peer1 = std::env::temp_dir().join(format!("peer-ca1-test-{}.pem", std::process::id()));
|
||||
// A leftover peer-CA file + the env var that used to name it. Both
|
||||
// must now be inert: the swarm root rides in the hive's own trust
|
||||
// bundle, so nothing per-peer is embedded any more.
|
||||
let stale_peer =
|
||||
std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id()));
|
||||
std::fs::write(
|
||||
&peer0,
|
||||
&stale_peer,
|
||||
"-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n",
|
||||
)
|
||||
.expect("write peer CA 0");
|
||||
std::fs::write(
|
||||
&peer1,
|
||||
"-----BEGIN CERTIFICATE-----\np1\n-----END CERTIFICATE-----\n",
|
||||
)
|
||||
.expect("write peer CA 1");
|
||||
let peer_paths = format!("{}:{}", peer0.display(), peer1.display());
|
||||
.expect("write stale peer CA");
|
||||
|
||||
// All env mutations are serialised within this one test (no other
|
||||
// test asserts on these vars), restored before returning.
|
||||
|
|
@ -2191,38 +2180,47 @@ mod tests {
|
|||
std::env::set_var("HIVE_TLS_CA_PATH", &ca_file);
|
||||
}
|
||||
let with_ca = render();
|
||||
// Hive CA + peer CAs: the list carries all three, hive CA first.
|
||||
unsafe {
|
||||
std::env::set_var("HIVE_PEER_CA_PATHS", &peer_paths);
|
||||
std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string());
|
||||
}
|
||||
let with_peers = render();
|
||||
// Peers only (this hive on ACME, federating with self-signed peers).
|
||||
let with_stale_peer_env = render();
|
||||
// The stale var alone, with no hive CA: must produce nothing.
|
||||
unsafe {
|
||||
std::env::remove_var("HIVE_TLS_CA_PATH");
|
||||
}
|
||||
let peers_only = render();
|
||||
let stale_peer_env_only = render();
|
||||
unsafe {
|
||||
std::env::remove_var("HIVE_PEER_CA_PATHS");
|
||||
}
|
||||
let without_ca = render();
|
||||
let _ = std::fs::remove_file(&ca_file);
|
||||
let _ = std::fs::remove_file(&peer0);
|
||||
let _ = std::fs::remove_file(&peer1);
|
||||
let _ = std::fs::remove_file(&stale_peer);
|
||||
|
||||
assert!(
|
||||
with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
|
||||
"CA cert must be wired into certificateFiles when signalled:\n{with_ca}"
|
||||
);
|
||||
// The regression guard, scoped to the CA list rather than the whole
|
||||
// render. Comparing the two flakes wholesale looks stronger and is
|
||||
// actually FLAKY: `cargo test` runs these in parallel threads and
|
||||
// sibling tests mutate process env (OTEL, forge URLs) between the
|
||||
// two `render()` calls, so a whole-output equality assertion fails
|
||||
// on changes that have nothing to do with this test.
|
||||
assert!(
|
||||
with_peers.contains(
|
||||
"security.pki.certificateFiles = [ ./hive-ca.pem ./peer-ca-0.pem ./peer-ca-1.pem ]"
|
||||
),
|
||||
"hive CA + peer CAs must all appear in the certificateFiles list:\n{with_peers}"
|
||||
with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
|
||||
"HIVE_PEER_CA_PATHS must not change the CA list — per-peer CA \
|
||||
embedding was replaced by the swarm root inside the hive's own \
|
||||
trust bundle:\n{with_stale_peer_env}"
|
||||
);
|
||||
assert!(
|
||||
peers_only
|
||||
.contains("security.pki.certificateFiles = [ ./peer-ca-0.pem ./peer-ca-1.pem ]"),
|
||||
"peer CAs must be trusted even when this hive has no self-signed CA:\n{peers_only}"
|
||||
!with_stale_peer_env.contains("peer-ca-"),
|
||||
"no peer-ca-<N>.pem may be embedded, however HIVE_PEER_CA_PATHS \
|
||||
is set:\n{with_stale_peer_env}"
|
||||
);
|
||||
assert!(
|
||||
!stale_peer_env_only.contains("security.pki.certificateFiles"),
|
||||
"a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \
|
||||
reference on its own:\n{stale_peer_env_only}"
|
||||
);
|
||||
assert!(
|
||||
!without_ca.contains("security.pki.certificateFiles"),
|
||||
|
|
|
|||
|
|
@ -1030,7 +1030,7 @@ fn is_broad_scope(scope: &LifecycleScope) -> bool {
|
|||
/// convention (gateway terminates TLS, so https).
|
||||
fn hive_urls() -> hive_host_sock::HiveUrls {
|
||||
// Treat an empty env value as unset everywhere — an empty domain would
|
||||
// otherwise render `swarm.peers."" = …` (invalid nix) and `https:///`.
|
||||
// otherwise render `swarm.hives."" = …` (invalid nix) and `https:///`.
|
||||
let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
|
||||
let domain = env("HYPERHIVE_HIVE_DOMAIN");
|
||||
hive_host_sock::HiveUrls {
|
||||
|
|
|
|||
Loading…
Reference in a new issue