Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f3da0a496 | ||
|
|
d547b521f6 |
12 changed files with 332 additions and 246 deletions
|
|
@ -83,7 +83,7 @@ The issuer is the **secret store's `pki` mount**, not a bare self-signed leaf. `
|
|||
|
||||
**Why an issuer, not a bare leaf**: a bare self-signed leaf is its own trust anchor, so every regeneration is a new anchor every consumer must re-trust, and an agent's build sets its trust store once, before any leaf exists. Agents and federation peers trust the store's root once; leaf rotation never breaks them.
|
||||
|
||||
**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. Agents (via `security.pki.certificateFiles`), the CI and forge containers and federating peers all read the bundle.
|
||||
**What consumers trust**: `trust-bundle.pem` in the same state dir. It carries the store's `pki` root ([`swarm/ca.md`](../swarm/ca.md#the-trust-bundle)). nginx serves the leaf alone, since its issuer is a self-signed root. The CI and forge containers and federating peers read the bundle. Agents fetch the root from the store themselves ([`swarm/ca.md`](../swarm/ca.md#distributing-the-root)).
|
||||
|
||||
**Why on by default**: matrix-dart-sdk (FluffyChat's SDK) fetches `https://<host>/.well-known/matrix/client` and never falls back to plain http, so without TLS the browser client can't bootstrap.
|
||||
|
||||
|
|
|
|||
|
|
@ -240,14 +240,10 @@ naming, because it's subtle and it bites at a distance:
|
|||
> the store's root.** The derivation reads `security.pki.certificateFiles`
|
||||
> inside itself; the root doesn't exist there.
|
||||
|
||||
Two consumers, and only one of them is fine:
|
||||
Two consumers, both reading the root at runtime:
|
||||
|
||||
- **`hive-tls.nix` covers agents.** It folds the root into this hive's
|
||||
`trust-bundle.pem`, `glue-hive-tls-controllers.nix` hands hive-c0re that
|
||||
path as `HIVE_TLS_CA_PATH`, and the meta-flake renderer embeds that one
|
||||
file next to every agent's flake. The bundle is the runtime-to-build-time
|
||||
bridge.
|
||||
- **The Matrix container crosses the same bridge**, via the shared
|
||||
- **Agents fetch it themselves.** `hive-agent-trust-bundle.service` (`nix/agent-modules/hive-ca.nix`) reads `<mount>/ca/pem` from the store with the agent's own store certificate, then writes `/run/hive-agent-trust-bundle/bundle.pem` (system CAs plus the root) and `root.pem` (the root alone). The agent's units point `SSL_CERT_FILE` at the first (Rust, Go and OpenSSL clients) and `NODE_EXTRA_CA_CERTS` at the second (Node and Claude Code), and order after the unit with `Requires=`. The meta flake names no CA file, and a timer re-reads the root hourly. A consumer reads it at start, so a rotated root reaches an agent unit at its next start.
|
||||
- **The Matrix container gets a bind mount**, via the shared
|
||||
`lib/hive-ca-trust.nix` helper that `hive-forge` and `swarm-ci-runner`
|
||||
use: the bundle is bind-mounted read-only into the container, and a
|
||||
oneshot concatenates it with the system CAs before tuwunel starts.
|
||||
|
|
|
|||
|
|
@ -590,8 +590,8 @@ The store's `swarm-services` role issues them instead, from the root inside its
|
|||
certificate is the only anchor the swarm has. Each hive fetches the certificate
|
||||
from the store at boot, so nothing build-time can name it — the system reads
|
||||
`security.pki.certificateFiles`' value at build time, which makes it the wrong
|
||||
tool here. Trust reaches containers through a bind-mounted bundle assembled at
|
||||
boot instead.
|
||||
tool here. Host containers get a bind-mounted bundle assembled at boot; an
|
||||
agent fetches the root itself with its own store certificate.
|
||||
|
||||
## Adding a secret
|
||||
|
||||
|
|
|
|||
|
|
@ -100,15 +100,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default();
|
||||
let initial = !dir.join(".git").exists();
|
||||
|
||||
// Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in
|
||||
// lockstep with its host source so the build-time `certificateFiles`
|
||||
// list render_flake emits always resolves. Empty when the hive has no
|
||||
// trust bundle.
|
||||
let (ca_files, ca_changed) = ca_embed_state(&dir);
|
||||
|
||||
// Skip only when both the flake AND the embedded CA are unchanged — a
|
||||
// CA rotation with an otherwise-identical flake must still re-commit.
|
||||
if !initial && on_disk == new_flake && !ca_changed {
|
||||
if !initial && on_disk == new_flake {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
|
@ -130,10 +122,9 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
std::fs::write(&flake_path, &new_flake)
|
||||
.with_context(|| format!("write {}", flake_path.display()))?;
|
||||
|
||||
// Materialise the embedded CA list next to flake.nix + drop any stale
|
||||
// CA file; `ca_touched` is every filename written or removed, staged
|
||||
// for commit below. Public CA certs only; no private key is embedded.
|
||||
let ca_touched = materialise_ca_files(&dir, &ca_files)?;
|
||||
// Agents fetch the swarm root from the store at boot, so no CA file sits
|
||||
// next to the flake; drop the ones earlier renders left behind.
|
||||
let ca_touched = remove_stale_ca_files(&dir);
|
||||
|
||||
// Reconcile topology.json against the live agent set — adds
|
||||
// newly-spawned agents and drops removed ones. Idempotent; when
|
||||
|
|
@ -175,8 +166,7 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
// contain '/flake.nix'". Lock then commit once with both
|
||||
// flake.nix and flake.lock — single commit per change.
|
||||
git(&dir, &["add", "flake.nix"]).await?;
|
||||
// Stage every embedded CA file we wrote or removed (trust bundle +
|
||||
// stale peer CAs). `git add <path>` stages a deletion when the path is
|
||||
// Stage every stale CA file removed above. `git add <path>` stages a deletion when the path is
|
||||
// tracked and now gone; best-effort so the never-tracked-and-absent case
|
||||
// (pathspec mismatch) is a harmless no-op.
|
||||
for name in &ca_touched {
|
||||
|
|
@ -204,9 +194,8 @@ pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
|
|||
.filter_map(|f| match f.as_str() {
|
||||
"flake.nix" => Some("flake"),
|
||||
"flake.lock" => Some("lock"),
|
||||
"hive-ca.pem" => Some("hive-ca"),
|
||||
// Only ever a REMOVAL now; see `is_embedded_ca_name`.
|
||||
f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"),
|
||||
// Only ever a removal; see `remove_stale_ca_files`.
|
||||
f if is_stale_ca_name(f) => Some("stale-ca"),
|
||||
"topology.json" => Some("topology"),
|
||||
_ => None,
|
||||
})
|
||||
|
|
@ -394,6 +383,9 @@ const FORWARDED_VARS: &[&str] = &[
|
|||
// hive's store environment wholesale — the address is the only part of it
|
||||
// an agent may have, the certificate beside it being the hive's own.
|
||||
"HIVE_AGENT_BAO_ADDR",
|
||||
// The mount path of the store's services `pki` engine, whose root an agent
|
||||
// fetches to trust the swarm. Forwarded with the address, under the same gate.
|
||||
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT",
|
||||
// Where an agent's `swarm-logs` queries, computed by the host for the
|
||||
// same reason as the two above: it is a gateway address, and a container
|
||||
// cannot derive it from anything it holds.
|
||||
|
|
@ -437,6 +429,12 @@ const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[
|
|||
// environment, so an agent whose option says no store and whose env says
|
||||
// otherwise would have a check that is simply not there.
|
||||
("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"),
|
||||
// Read at build time to bake the root-fetch URL into
|
||||
// `nix/agent-modules/hive-ca.nix`'s unit.
|
||||
(
|
||||
"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT",
|
||||
"services.hyperhive.agent.bao.servicesPkiMount",
|
||||
),
|
||||
// Read at build time to decide whether `swarm-logs` is installed at all —
|
||||
// see `nix/agent-modules/logs.nix`. Both halves wired for this map's own
|
||||
// reason: an agent whose option says no log store and whose env says
|
||||
|
|
@ -537,30 +535,6 @@ fn forwarded_env_vars() -> Vec<(&'static str, String)> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Filename the hive's own trust anchors are embedded under at the
|
||||
/// meta-flake root — the only entry of the embedded-CA list
|
||||
/// `render_flake` references in `security.pki.certificateFiles` (see
|
||||
/// `embedded_ca_files`).
|
||||
const HIVE_CA_FILE: &str = "hive-ca.pem";
|
||||
|
||||
/// Host path of the hive's TLS trust anchors, when self-signed TLS is
|
||||
/// active. The hyperhive TLS module sets `HIVE_TLS_CA_PATH` in hive-c0re's env
|
||||
/// (to `<deploy.hive-controller.tls.stateDir>/trust-bundle.pem`) whenever the
|
||||
/// gateway serves a leaf issued by the secret store's `pki` mount. The
|
||||
/// file holds that mount's root, the one anchor every hive's gateway leaf
|
||||
/// chains to, and is copied verbatim, which `security.pki.certificateFiles`
|
||||
/// accepts; nothing here parses it. Returns `Some(path)` only when the var
|
||||
/// is set AND the cert exists on disk — so render + write stay consistent
|
||||
/// (we never emit a `certificateFiles` reference to a file we didn't
|
||||
/// embed). Only the public cert is ever read here.
|
||||
fn hive_ca_source() -> Option<String> {
|
||||
let path = std::env::var("HIVE_TLS_CA_PATH").ok()?;
|
||||
if path.is_empty() || !std::path::Path::new(&path).is_file() {
|
||||
return None;
|
||||
}
|
||||
Some(path)
|
||||
}
|
||||
|
||||
/// Extra `OTEL_RESOURCE_ATTRIBUTES` entries for every agent, read off
|
||||
/// hive-c0re's own unit env
|
||||
/// (`services.hyperhive.deploy.hive-controller.otel.extraResourceAttributes`).
|
||||
|
|
@ -574,105 +548,32 @@ fn otel_extra_resource_attributes() -> Option<String> {
|
|||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
/// The ordered set of CA certs embedded next to the meta flake, as
|
||||
/// `(filename, host_source_path)` — now just the hive's own trust
|
||||
/// anchors as `hive-ca.pem`, when self-signed TLS is active.
|
||||
///
|
||||
/// That single file carries the store's `pki` root (the hyperhive TLS
|
||||
/// module writes it into the trust bundle), so every hive's gateway leaf
|
||||
/// validates from it and nothing is embedded per peer.
|
||||
///
|
||||
/// `render_flake` emits exactly these filenames into
|
||||
/// `security.pki.certificateFiles` and `sync_agents` materialises them,
|
||||
/// so the rendered reference and the embedded files always agree.
|
||||
fn embedded_ca_files() -> Vec<(String, String)> {
|
||||
let mut out = Vec::new();
|
||||
if let Some(p) = hive_ca_source() {
|
||||
out.push((HIVE_CA_FILE.to_owned(), p));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Write each desired embedded CA file next to `flake.nix` and remove
|
||||
/// any stale one (self-signed TLS turned off, or a leftover peer CA),
|
||||
/// so the flake never references a file we didn't write. Returns every
|
||||
/// filename written or removed, for the caller to stage. The public CA
|
||||
/// certs only; no private key is ever embedded.
|
||||
fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result<Vec<String>> {
|
||||
let desired: std::collections::HashSet<&str> =
|
||||
ca_files.iter().map(|(n, _)| n.as_str()).collect();
|
||||
let mut touched: Vec<String> = Vec::new();
|
||||
/// Remove the CA files earlier renders embedded next to `flake.nix`
|
||||
/// (`hive-ca.pem`, and the per-peer `peer-ca-<N>.pem` before it), returning
|
||||
/// every filename removed so the caller stages the deletion. Nothing writes
|
||||
/// one now: agents fetch the swarm root from the store at boot, so a file left
|
||||
/// here is referenced by nothing.
|
||||
fn remove_stale_ca_files(dir: &Path) -> Vec<String> {
|
||||
let mut removed = Vec::new();
|
||||
if let Ok(entries) = std::fs::read_dir(dir) {
|
||||
for e in entries.flatten() {
|
||||
let fname = e.file_name();
|
||||
let Some(name) = fname.to_str() else { continue };
|
||||
if is_embedded_ca_name(name) && !desired.contains(name) {
|
||||
let _ = std::fs::remove_file(dir.join(name));
|
||||
touched.push(name.to_owned());
|
||||
if is_stale_ca_name(name) && std::fs::remove_file(dir.join(name)).is_ok() {
|
||||
removed.push(name.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (name, content) in ca_files {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?;
|
||||
touched.push(name.clone());
|
||||
}
|
||||
Ok(touched)
|
||||
removed
|
||||
}
|
||||
|
||||
/// True for a filename the embedded-CA machinery owns. Lets
|
||||
/// `sync_agents` find stale CA files to clean up (a CA dropped from
|
||||
/// config) without touching unrelated meta files.
|
||||
///
|
||||
/// ⚠️ Still matches `peer-ca-<N>.pem`, which `embedded_ca_files` no
|
||||
/// longer produces — deliberately. Cleanup is driven by
|
||||
/// "recognised but not desired", so this arm is exactly what removes a
|
||||
/// per-peer CA file still sitting at a meta root. Drop it and those files
|
||||
/// are orphaned at every meta root forever, referenced by nothing and
|
||||
/// cleaned by nobody.
|
||||
fn is_embedded_ca_name(name: &str) -> bool {
|
||||
name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name))
|
||||
}
|
||||
|
||||
/// True when `name` ends in a `.pem` extension (case-insensitive). Split
|
||||
/// out so the embedded-CA filename checks share one spelling and dodge
|
||||
/// clippy's case-sensitive-extension lint.
|
||||
fn has_pem_ext(name: &str) -> bool {
|
||||
Path::new(name)
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("pem"))
|
||||
}
|
||||
|
||||
/// Embedded-CA state for the meta repo: `(desired_files, changed)`.
|
||||
/// `desired_files` is `(filename, contents)` for every CA that should sit
|
||||
/// next to flake.nix. `changed` is true when the on-disk set differs in
|
||||
/// any way — the file's contents changed, a CA appeared, or a
|
||||
/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is
|
||||
/// no longer wanted (stale, to be removed). Drives both the re-commit
|
||||
/// decision and the materialise/cleanup in `sync_agents`, so a CA
|
||||
/// rotation re-commits even when the flake itself is byte-identical.
|
||||
fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) {
|
||||
let desired: Vec<(String, String)> = embedded_ca_files()
|
||||
.into_iter()
|
||||
.filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c)))
|
||||
.collect();
|
||||
let desired_names: std::collections::HashSet<&str> =
|
||||
desired.iter().map(|(n, _)| n.as_str()).collect();
|
||||
|
||||
let mut changed = desired.iter().any(|(name, content)| {
|
||||
std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content
|
||||
});
|
||||
|
||||
// A previously-embedded CA file no longer wanted → stale (removal is
|
||||
// a change even when every desired file already matches on disk).
|
||||
if !changed && let Ok(entries) = std::fs::read_dir(dir) {
|
||||
changed = entries.flatten().any(|e| {
|
||||
e.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name))
|
||||
});
|
||||
}
|
||||
(desired, changed)
|
||||
/// True for the filenames [`remove_stale_ca_files`] owns.
|
||||
fn is_stale_ca_name(name: &str) -> bool {
|
||||
name == "hive-ca.pem"
|
||||
|| (name.starts_with("peer-ca-")
|
||||
&& Path::new(name)
|
||||
.extension()
|
||||
.is_some_and(|ext| ext.eq_ignore_ascii_case("pem")))
|
||||
}
|
||||
|
||||
/// Resolve every custom agent's [`AgentPin`] and return `agents` with it set.
|
||||
|
|
@ -862,26 +763,6 @@ fn render_flake(
|
|||
" services.hyperhive.agent.claudeCodePath = \"{path}\";"
|
||||
);
|
||||
}
|
||||
// CA trust: embed the hive's trust bundle so each agent validates
|
||||
// the store's `pki` root at build time (one file; see
|
||||
// `hive_ca_source`). `certificateFiles` is build-time, so the certs
|
||||
// travel with the flake source: `sync_agents` writes `./hive-ca.pem`
|
||||
// next to flake.nix and stages it. Only public CA certs are embedded;
|
||||
// no private key ever leaves the host. The filename list matches
|
||||
// `sync_agents` exactly (both derive it from `embedded_ca_files`), so
|
||||
// we never reference a file we didn't embed; emitted only when the
|
||||
// list is non-empty.
|
||||
let ca_refs: Vec<String> = embedded_ca_files()
|
||||
.into_iter()
|
||||
.map(|(name, _)| format!("./{name}"))
|
||||
.collect();
|
||||
if !ca_refs.is_empty() {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
" security.pki.certificateFiles = [ {} ];",
|
||||
ca_refs.join(" ")
|
||||
);
|
||||
}
|
||||
// Extra OTEL resource labels (`deploy.hive-controller.otel.*`) are the one
|
||||
// OTEL value rendered here; enablement and the collector endpoint are
|
||||
// agent-module defaults.
|
||||
|
|
@ -2005,24 +1886,13 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn render_flake_embeds_hive_ca_when_signalled() {
|
||||
// When the hyperhive TLS module signals a trust bundle via
|
||||
// HIVE_TLS_CA_PATH (and the cert exists), the agent module must
|
||||
// trust it at build time via security.pki.certificateFiles. Absent
|
||||
// the signal, no reference is emitted (so the flake doesn't point at
|
||||
// a file that was never embedded).
|
||||
fn render_flake_embeds_no_ca() {
|
||||
// Agents fetch the swarm root from the store at boot; the flake names
|
||||
// no CA file whatever the host env says, including the variable
|
||||
// hive-c0re used to embed from.
|
||||
//
|
||||
// Serialised against every other env-mutating test in the crate. Taken
|
||||
// before the fixture files so the guard covers the whole window in
|
||||
// which HIVE_TLS_CA_PATH / HIVE_PEER_CA_PATHS are perturbed.
|
||||
// Serialised against every other env-mutating test in the crate.
|
||||
let _env = crate::test_env::lock();
|
||||
let ca_file = std::env::temp_dir().join(format!("hive-ca-test-{}.pem", std::process::id()));
|
||||
std::fs::write(
|
||||
&ca_file,
|
||||
"-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n",
|
||||
)
|
||||
.expect("write temp CA");
|
||||
|
||||
let render = || {
|
||||
render(
|
||||
"git+https://forge.example/hyperhive/hyperhive.git?rev=0123abcd",
|
||||
|
|
@ -2031,70 +1901,59 @@ mod tests {
|
|||
&[sample_spec("alice", 9001)],
|
||||
)
|
||||
};
|
||||
|
||||
// A leftover peer-CA file + the env var that names it. Both must be
|
||||
// inert: the store's root rides in the hive's own trust bundle, so
|
||||
// nothing per-peer is embedded.
|
||||
let stale_peer =
|
||||
std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id()));
|
||||
std::fs::write(
|
||||
&stale_peer,
|
||||
"-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n",
|
||||
)
|
||||
.expect("write stale peer CA");
|
||||
|
||||
// SAFETY: serialised by the guard at the top of this test; restored
|
||||
// before returning.
|
||||
unsafe {
|
||||
std::env::remove_var("HIVE_PEER_CA_PATHS");
|
||||
std::env::set_var("HIVE_TLS_CA_PATH", &ca_file);
|
||||
std::env::set_var("HIVE_TLS_CA_PATH", "/nonexistent/trust-bundle.pem");
|
||||
}
|
||||
let with_ca = render();
|
||||
unsafe {
|
||||
std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string());
|
||||
}
|
||||
let with_stale_peer_env = render();
|
||||
// The stale var alone, with no trust bundle: must produce nothing.
|
||||
let flake = render();
|
||||
unsafe {
|
||||
std::env::remove_var("HIVE_TLS_CA_PATH");
|
||||
}
|
||||
let stale_peer_env_only = render();
|
||||
unsafe {
|
||||
std::env::remove_var("HIVE_PEER_CA_PATHS");
|
||||
// Presence control: the render does carry per-agent module config, so
|
||||
// the absences below are not an empty render.
|
||||
assert!(
|
||||
flake.contains("services.hyperhive.agent."),
|
||||
"per-agent module block missing:\n{flake}"
|
||||
);
|
||||
for needle in ["hive-ca.pem", "certificateFiles", "peer-ca-"] {
|
||||
assert!(
|
||||
!flake.contains(needle),
|
||||
"the flake must not name {needle}:\n{flake}"
|
||||
);
|
||||
}
|
||||
let without_ca = render();
|
||||
let _ = std::fs::remove_file(&ca_file);
|
||||
let _ = std::fs::remove_file(&stale_peer);
|
||||
}
|
||||
|
||||
assert!(
|
||||
with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
|
||||
"CA cert must be wired into certificateFiles when signalled:\n{with_ca}"
|
||||
#[test]
|
||||
fn stale_ca_files_are_removed_and_nothing_else() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
for name in ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM", "flake.nix"] {
|
||||
std::fs::write(dir.path().join(name), "x").expect("write");
|
||||
}
|
||||
// Not CA files this machinery owns, though they look similar.
|
||||
for name in ["peer-ca-0.txt", "other-ca.pem"] {
|
||||
std::fs::write(dir.path().join(name), "x").expect("write");
|
||||
}
|
||||
let mut removed = remove_stale_ca_files(dir.path());
|
||||
removed.sort();
|
||||
assert_eq!(removed, ["hive-ca.pem", "peer-ca-0.pem", "peer-ca-1.PEM"]);
|
||||
for kept in ["flake.nix", "peer-ca-0.txt", "other-ca.pem"] {
|
||||
assert!(dir.path().join(kept).exists(), "{kept} must survive");
|
||||
}
|
||||
assert!(remove_stale_ca_files(dir.path()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn services_pki_mount_is_forwarded_as_both_env_and_option() {
|
||||
assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_BAO_SERVICES_PKI_MOUNT"));
|
||||
let mut out = String::new();
|
||||
push_forwarded_var_options(
|
||||
&mut out,
|
||||
&[("HIVE_AGENT_BAO_SERVICES_PKI_MOUNT", "pki".to_string())],
|
||||
);
|
||||
// The regression guard, scoped to the CA list rather than the whole
|
||||
// render. Comparing the two flakes wholesale looks stronger and is
|
||||
// actually FLAKY: `cargo test` runs these in parallel threads and
|
||||
// sibling tests mutate process env (OTEL, forge URLs) between the
|
||||
// two `render()` calls, so a whole-output equality assertion fails
|
||||
// on changes that have nothing to do with this test.
|
||||
assert!(
|
||||
with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
|
||||
"HIVE_PEER_CA_PATHS must not change the CA list — the store's \
|
||||
root inside the hive's own trust bundle is the only anchor \
|
||||
embedded:\n{with_stale_peer_env}"
|
||||
);
|
||||
assert!(
|
||||
!with_stale_peer_env.contains("peer-ca-"),
|
||||
"no peer-ca-<N>.pem may be embedded, however HIVE_PEER_CA_PATHS \
|
||||
is set:\n{with_stale_peer_env}"
|
||||
);
|
||||
assert!(
|
||||
!stale_peer_env_only.contains("security.pki.certificateFiles"),
|
||||
"a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \
|
||||
reference on its own:\n{stale_peer_env_only}"
|
||||
);
|
||||
assert!(
|
||||
!without_ca.contains("security.pki.certificateFiles"),
|
||||
"no certificateFiles reference without any CA signal:\n{without_ca}"
|
||||
assert_eq!(
|
||||
out,
|
||||
" services.hyperhive.agent.bao.servicesPkiMount = \"pki\";\n"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ reqwest = { workspace = true, features = [
|
|||
# `rustls` carries the platform verifier, which reads the OS trust store
|
||||
# (and honours `SSL_CERT_FILE`), so hive-forge validates the hive
|
||||
# gateway's self-signed leaf once the store's root is in the agent's trust
|
||||
# store — embedded into agent flakes via `security.pki.certificateFiles`
|
||||
# — while public CAs still validate.
|
||||
# store (an agent fetches it from the secret store at boot and points
|
||||
# `SSL_CERT_FILE` at it) while public CAs still validate.
|
||||
#
|
||||
# The separate `rustls-native-certs` / `webpki-roots` features this entry
|
||||
# used to name were 0.13.1-era and no longer exist; naming a feature that
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ in
|
|||
./frontend.nix
|
||||
./github.nix
|
||||
./github-token.nix
|
||||
./hive-ca.nix
|
||||
./hive-facts.nix
|
||||
./logs.nix
|
||||
./matrix.nix
|
||||
|
|
|
|||
196
nix/agent-modules/hive-ca.nix
Normal file
196
nix/agent-modules/hive-ca.nix
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
# The swarm's root certificate in this agent's trust, fetched from the store
|
||||
# at boot by the agent itself.
|
||||
#
|
||||
# The root is the store's services `pki` mount's CA. It does not exist when the
|
||||
# agent is built, so the meta flake names no CA file: this unit reads it from
|
||||
# the store with the agent's own certificate (./bao.nix keeps it current) and
|
||||
# writes two files, both public:
|
||||
#
|
||||
# - `bundle`: the system CAs plus the swarm root. `SSL_CERT_FILE` replaces the
|
||||
# trust store of Rust (rustls platform verifier), Go and OpenSSL clients, so
|
||||
# it carries both halves.
|
||||
# - `root`: the swarm root alone, for `NODE_EXTRA_CA_CERTS`, which adds to
|
||||
# Node's store (Claude Code).
|
||||
#
|
||||
# A process reads its trust when it starts, so a rotated root reaches the
|
||||
# consumers below at their next start. They `Requires=` this unit, which keeps
|
||||
# one from running with a store that lacks the root.
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.hyperhive.agent.bao;
|
||||
agentCfg = config.services.hyperhive.agent;
|
||||
|
||||
agentName = agentCfg.user.name;
|
||||
|
||||
# The server CA id ./bao.nix loads.
|
||||
serverCaCredential = "hive-agent-bao-server-ca";
|
||||
|
||||
unitName = "hive-agent-trust-bundle";
|
||||
runtimeDir = unitName;
|
||||
bundleFile = "/run/${runtimeDir}/bundle.pem";
|
||||
rootFile = "/run/${runtimeDir}/root.pem";
|
||||
|
||||
systemBundle = "/etc/ssl/certs/ca-certificates.crt";
|
||||
|
||||
configured = cfg.addr != null;
|
||||
|
||||
storeRetry = import ../host-modules/lib/store-retry.nix { };
|
||||
|
||||
# Units that dial a swarm service over TLS. The first group is defined
|
||||
# whenever the agent module is; the rest exist under the same condition as
|
||||
# their definitions, because ordering on a unit that is not defined would
|
||||
# generate an inert one.
|
||||
consumers = [
|
||||
"hive-agent"
|
||||
"hive-bash-daemon"
|
||||
"hive-subagent-daemon"
|
||||
"hive-mcp-http"
|
||||
"hive-forge-notify"
|
||||
]
|
||||
++ lib.optional (agentCfg.matrixAccounts != { }) "hive-matrix-daemon"
|
||||
++ lib.optional (agentCfg.github.enable) "hive-github-notify"
|
||||
++ lib.optional (agentCfg.icon != null && agentCfg.forge.url != null) "forge-avatar-sync";
|
||||
|
||||
trustEnv = {
|
||||
SSL_CERT_FILE = bundleFile;
|
||||
NODE_EXTRA_CA_CERTS = rootFile;
|
||||
};
|
||||
in
|
||||
{
|
||||
options.services.hyperhive.agent.bao.servicesPkiMount = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "pki";
|
||||
example = "pki";
|
||||
description = ''
|
||||
Mount path of the store's services `pki` engine, whose root is the one
|
||||
anchor every swarm service certificate chains to. Set by the generated
|
||||
meta flake from the host's
|
||||
`services.hyperhive.deploy.bao.servicesPkiMountPath`.
|
||||
`${unitName}.service` reads `<mount>/ca/pem` from it.
|
||||
'';
|
||||
};
|
||||
|
||||
config = lib.mkIf configured {
|
||||
# Every process, including a bash task's children, resolves trust through
|
||||
# these, as it resolves `HIVE_FORGE_TOKEN_FILE`.
|
||||
systemd.globalEnvironment = trustEnv;
|
||||
environment.variables = trustEnv;
|
||||
|
||||
systemd.services = {
|
||||
${unitName} = {
|
||||
description = "fetch the swarm root certificate from the secret store";
|
||||
after = [
|
||||
"network.target"
|
||||
"hive-agent-bao-identity.service"
|
||||
];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.openssl
|
||||
];
|
||||
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
||||
serviceConfig = storeRetry.serviceConfig // {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = false;
|
||||
TimeoutStartSec = 60;
|
||||
User = agentName;
|
||||
Group = agentName;
|
||||
RuntimeDirectory = runtimeDir;
|
||||
# Public certificates, read by every consumer's user.
|
||||
RuntimeDirectoryMode = "0755";
|
||||
RuntimeDirectoryPreserve = "yes";
|
||||
UMask = "0022";
|
||||
LoadCredential = [ serverCaCredential ];
|
||||
};
|
||||
# Overrides the global value: this unit runs before the bundle exists,
|
||||
# and its own TLS is checked against the credential below or the
|
||||
# system bundle.
|
||||
environment = {
|
||||
SSL_CERT_FILE = lib.mkForce systemBundle;
|
||||
BAO_ADDR = cfg.addr;
|
||||
BAO_CLIENT_CERT = cfg.clientCertFile;
|
||||
BAO_CLIENT_KEY = cfg.clientKeyFile;
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
for f in "$BAO_CLIENT_CERT" "$BAO_CLIENT_KEY"; do
|
||||
if [ ! -s "$f" ]; then
|
||||
echo "this agent has no store identity at $f, so it cannot fetch the swarm root certificate." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
cacert=
|
||||
if [ -s "$CREDENTIALS_DIRECTORY/${serverCaCredential}" ]; then
|
||||
cacert="$CREDENTIALS_DIRECTORY/${serverCaCredential}"
|
||||
echo "trusting bao via the delivered server CA credential" >&2
|
||||
else
|
||||
echo "no bao server CA credential delivered; trusting the system bundle" >&2
|
||||
fi
|
||||
|
||||
dir=/run/${runtimeDir}
|
||||
err=$(mktemp -p "$dir" err.XXXXXX)
|
||||
raw=$(mktemp -p "$dir" raw.XXXXXX)
|
||||
root=$(mktemp -p "$dir" root.XXXXXX)
|
||||
bundle=$(mktemp -p "$dir" bundle.XXXXXX)
|
||||
trap 'rm -f "$err" "$raw" "$root" "$bundle"' EXIT
|
||||
|
||||
# The listener demands a client certificate and the endpoint takes no
|
||||
# token, so no login precedes this.
|
||||
if ! curl --fail --silent --show-error \
|
||||
--connect-timeout 10 --max-time 30 \
|
||||
''${cacert:+--cacert "$cacert"} \
|
||||
--cert "$BAO_CLIENT_CERT" --key "$BAO_CLIENT_KEY" \
|
||||
--output "$raw" \
|
||||
${lib.escapeShellArg "${cfg.addr}/v1/${cfg.servicesPkiMount}/ca/pem"} 2>"$err"; then
|
||||
echo "could not read the store's ${cfg.servicesPkiMount} root at $BAO_ADDR." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Re-encoded rather than copied: it fails on a body that is not a
|
||||
# certificate and ends the file in the newline the concatenation
|
||||
# below relies on.
|
||||
if ! openssl x509 -in "$raw" -out "$root" 2>"$err"; then
|
||||
echo "the store's ${cfg.servicesPkiMount} root does not parse as a certificate." >&2
|
||||
cat "$err" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat ${systemBundle} "$root" >"$bundle"
|
||||
|
||||
# Neither file moves unless it changed, so a reader never sees a half
|
||||
# write and the timer's rerun is silent.
|
||||
if cmp -s "$bundle" ${bundleFile} && cmp -s "$root" ${rootFile}; then
|
||||
echo "the store's ${cfg.servicesPkiMount} root is unchanged"
|
||||
exit 0
|
||||
fi
|
||||
chmod 0644 "$root" "$bundle"
|
||||
mv -f "$root" ${rootFile}
|
||||
mv -f "$bundle" ${bundleFile}
|
||||
echo "wrote the swarm root certificate from the store's ${cfg.servicesPkiMount} mount"
|
||||
'';
|
||||
};
|
||||
}
|
||||
// lib.genAttrs consumers (_: {
|
||||
after = [ "${unitName}.service" ];
|
||||
requires = [ "${unitName}.service" ];
|
||||
});
|
||||
|
||||
systemd.timers.${unitName} = {
|
||||
description = "re-read the swarm root certificate from the secret store";
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnUnitInactiveSec = "1h";
|
||||
RandomizedDelaySec = "1min";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -2,13 +2,10 @@
|
|||
#
|
||||
# ONE PAIRING PER FILE — hive-c0re and the swarm controller ← the trust bundle
|
||||
# ./hive-tls.nix assembles. Deleting this leaves daemons that trust only the
|
||||
# platform roots: hive-c0re's lifecycle cannot hand agents the bundle, and the
|
||||
# swarm-queue clients die at `invalid peer certificate: UnknownIssuer` against
|
||||
# an authelia signed by the swarm CA.
|
||||
# platform roots: the swarm-queue clients die at
|
||||
# `invalid peer certificate: UnknownIssuer` against an authelia signed by the
|
||||
# swarm CA.
|
||||
#
|
||||
# `HIVE_TLS_CA_PATH` signals the hive-c0re lifecycle that a trust bundle exists:
|
||||
# it bind-mounts the file (read-only, public certs ONLY — never a key) into each
|
||||
# agent container, and the meta flake wires the per-agent trust bundle.
|
||||
# `*_OIDC_CA_FILE` names the same anchor for the swarm-queue clients that mint a
|
||||
# token from authelia over TLS. `swarm-queue-client` builds a bare
|
||||
# `reqwest::Client`, so it trusts only the platform roots without it. Where the
|
||||
|
|
@ -43,7 +40,6 @@ in
|
|||
lib.mkMerge [
|
||||
(lib.mkIf deployCfg.hive-controller.enable {
|
||||
systemd.services.hive-c0re.environment = {
|
||||
HIVE_TLS_CA_PATH = trustBundle;
|
||||
HIVE_C0RE_OIDC_CA_FILE = trustBundle;
|
||||
};
|
||||
})
|
||||
|
|
|
|||
|
|
@ -278,6 +278,9 @@ in
|
|||
# environment wholesale would inherit the hive's certificate paths with it,
|
||||
# and those name a credential no agent may present.
|
||||
HIVE_AGENT_BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
||||
# The mount whose root an agent fetches to trust the swarm's services; the
|
||||
# same value ../hive-tls.nix reads the root from.
|
||||
HIVE_AGENT_BAO_SERVICES_PKI_MOUNT = baoDeploy.servicesPkiMountPath;
|
||||
# `%d`, not the paths themselves: the key is `0600` root-owned and this
|
||||
# daemon runs as hive-core, so it never gets read access to the original.
|
||||
# See the LoadCredential in ./default.nix.
|
||||
|
|
|
|||
|
|
@ -47,6 +47,38 @@ let
|
|||
in
|
||||
let
|
||||
cases = [
|
||||
{
|
||||
# ../agent-modules/hive-ca.nix. Presence arm first (the unit, the
|
||||
# timer, the trust variables, and every consumer ordered after the
|
||||
# unit and requiring it), then the absence arm for an agent with no
|
||||
# store, which has no root to fetch.
|
||||
name = "an agent with a store address fetches the swarm root and its consumers wait for it";
|
||||
ok =
|
||||
let
|
||||
svc = agentForgeBao.systemd.services;
|
||||
unit = "hive-agent-trust-bundle.service";
|
||||
bundle = "/run/hive-agent-trust-bundle/bundle.pem";
|
||||
consumers = [
|
||||
"hive-agent"
|
||||
"hive-bash-daemon"
|
||||
"hive-subagent-daemon"
|
||||
"hive-mcp-http"
|
||||
"hive-forge-notify"
|
||||
"forge-avatar-sync"
|
||||
];
|
||||
in
|
||||
svc ? hive-agent-trust-bundle
|
||||
&& agentForgeBao.systemd.timers ? hive-agent-trust-bundle
|
||||
&& lib.hasInfix "/v1/pki/ca/pem" svc.hive-agent-trust-bundle.script
|
||||
&& agentForgeBao.systemd.globalEnvironment.SSL_CERT_FILE == bundle
|
||||
&&
|
||||
agentForgeBao.systemd.globalEnvironment.NODE_EXTRA_CA_CERTS
|
||||
== "/run/hive-agent-trust-bundle/root.pem"
|
||||
&& lib.all (c: lib.elem unit svc.${c}.after && lib.elem unit svc.${c}.requires) consumers
|
||||
&& !(agentForgeNoBao.systemd.services ? hive-agent-trust-bundle)
|
||||
&& !(agentForgeNoBao.systemd.globalEnvironment ? SSL_CERT_FILE)
|
||||
&& !(lib.elem unit (agentForgeNoBao.systemd.services.hive-agent.after or [ ]));
|
||||
}
|
||||
{
|
||||
# The whole switch is the store address, as for the queue credential.
|
||||
name = "an agent with a store address fetches its forge token";
|
||||
|
|
|
|||
|
|
@ -317,6 +317,12 @@ let
|
|||
in
|
||||
e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
|
||||
}
|
||||
{
|
||||
name = "an agent is told the mount its swarm root is read from, only where it is told the store";
|
||||
ok =
|
||||
allLocal.systemd.services.hive-c0re.environment.HIVE_AGENT_BAO_SERVICES_PKI_MOUNT == "pki"
|
||||
&& !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_SERVICES_PKI_MOUNT);
|
||||
}
|
||||
{
|
||||
# A hive with no certificate of its own can collect no agent's identity,
|
||||
# so forwarding an address would name a store nothing in the container
|
||||
|
|
|
|||
|
|
@ -693,10 +693,7 @@ let
|
|||
let
|
||||
bundle = "/var/lib/hive-tls/trust-bundle.pem";
|
||||
envOf = m: unit: m.systemd.services.${unit}.environment or { };
|
||||
hiveVars = [
|
||||
"HIVE_TLS_CA_PATH"
|
||||
"HIVE_C0RE_OIDC_CA_FILE"
|
||||
];
|
||||
hiveVars = [ "HIVE_C0RE_OIDC_CA_FILE" ];
|
||||
anyOf =
|
||||
m: unit: vars:
|
||||
lib.any (v: (envOf m unit) ? ${v}) vars;
|
||||
|
|
|
|||
Loading…
Reference in a new issue