hyperhive/hive-c0re/src/meta.rs

//! Single hive-c0re-owned flake at `/var/lib/hyperhive/meta/` that
//! exports one `nixosConfiguration` per agent and drives the system-wide
//! deploy audit trail. Flow (`sync_agents`, `lock_update_for_rebuild`,
//! `lock_update_hyperhive`):
//! `docs/agent-lifecycle/config-changes.md::Meta flake`.

use std::path::Path;

use anyhow::{Context, Result, bail};
use tokio::process::Command;
use tokio::sync::Mutex;

use crate::coordinator::HiveEnv;
use crate::lifecycle;

const GIT_NAME: &str = "c0re";
const GIT_EMAIL: &str = "c0re@hyperhive.local";

/// Single-writer lock around every meta-repo operation. Git isn't
/// safe to drive from concurrent processes against the same `.git/`
/// — two simultaneous `git add` / `commit` invocations race on
/// `.git/index.lock`; if either dies before releasing, the lock
/// sticks and the next operation hits "another git process seems to
/// be running" until somebody `rm`s it manually. Holding this mutex
/// across each public function's git+nix calls makes parallel
/// rebuilds (`auto_update` + dashboard-triggered + apply-commit)
/// take turns instead of colliding.
static META_LOCK: Mutex<()> = Mutex::const_new(());

// Exclusivity for meta-repo *windows* that span multiple `META_LOCK`
// acquisitions is **not** a mutex in this module. `META_LOCK` above
// serializes individual git ops but cannot keep another op out of a window
// spanning several of them; that window is owned by the job queue instead,
// as `Resource::MetaWindow`, declared by every meta-mutating node kind. A
// resource can be held by a subtree root across its children, which a `MutexGuard`
// (bounded by one executor fn) cannot.

/// Where the manager sees this directory inside its container (RO bind).
pub const CONTAINER_MANAGER_META_MOUNT: &str = "/meta";

#[derive(Debug, Clone)]
pub struct AgentSpec {
    pub name: String,
    pub port: u16,
    /// The archetype the agent is built from (`crate::archetypes`), `None`
    /// for a custom agent built from its own config repo.
    pub archetype: Option<swarm_queue_client::agent_archetype::ArchetypeRef>,
}

/// Stage topology.json if it exists: it is regenerated by `reconcile`
/// whenever the agent set changed. Without staging, an
/// existing-but-untracked file shows up as untracked in the meta repo,
/// which can confuse nix's dirty-tree fetch. `git add` is a no-op when
/// content is unchanged.
async fn stage_generated_meta_files(dir: &std::path::Path) -> Result<()> {
    if crate::topology::topology_path().exists() {
        git(dir, &["add", "topology.json"]).await?;
    }
    Ok(())
}

/// Idempotently reconcile the meta repo with the current agent set.
/// First call inits the git repo, runs `nix flake lock`, and lands a
/// seed commit. Subsequent calls only touch `flake.nix` when the
/// rendered contents differ from disk; an unchanged `flake.nix` is a
/// no-op.
pub async fn sync_agents(hive: &HiveEnv, agents: &[AgentSpec]) -> Result<()> {
    let _guard = META_LOCK.lock().await;
    // Before anything is written: a hive without a forge URL would deploy a
    // whole fleet of agents that silently never log in, because the agent
    // option treats "unset" as "no forge configured" rather than erroring.
    // This is the layer that knows a forge is mandatory, so it says so here.
    require_service_urls(&forwarded_env_vars())?;
    let dir = crate::paths::meta_root();
    std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;

    let new_flake = render_hive_flake(hive, agents);
    let flake_path = dir.join("flake.nix");
    let on_disk = std::fs::read_to_string(&flake_path).unwrap_or_default();
    let initial = !dir.join(".git").exists();

    // Embedded-CA list: keep the `./hive-ca.pem` file at the meta root in
    // lockstep with its host source so the build-time `certificateFiles`
    // list render_flake emits always resolves. Empty when the hive has no
    // trust bundle.
    let (ca_files, ca_changed) = ca_embed_state(&dir);

    // Skip only when both the flake AND the embedded CA are unchanged — a
    // CA rotation with an otherwise-identical flake must still re-commit.
    if !initial && on_disk == new_flake && !ca_changed {
        return Ok(());
    }

    // Safety guard: refuse to write an empty agent list over a non-empty
    // on-disk flake. An empty `agents` slice is never intentional — it
    // means `nixos-container list` failed and the caller got an empty
    // fallback. Overwriting here would drop every agent from the meta
    // flake and trigger unnecessary (and potentially destructive) cascade
    // rebuilds. Callers that genuinely need to clear the agent list
    // (there are none today) must handle this case explicitly.
    if agents.is_empty() && !initial && !on_disk.is_empty() {
        tracing::warn!(
            "sync_agents: refusing to overwrite non-empty meta flake with empty agent list \
             (nixos-container list may have failed)"
        );
        return Ok(());
    }

    std::fs::write(&flake_path, &new_flake)
        .with_context(|| format!("write {}", flake_path.display()))?;

    // Materialise the embedded CA list next to flake.nix + drop any stale
    // CA file; `ca_touched` is every filename written or removed, staged
    // for commit below. Public CA certs only; no private key is embedded.
    let ca_touched = materialise_ca_files(&dir, &ca_files)?;

    // Reconcile topology.json against the live agent set — adds
    // newly-spawned agents and drops removed ones. Idempotent; when
    // nothing changed the file isn't touched.
    let agent_names: Vec<String> = agents.iter().map(|a| a.name.clone()).collect();
    let pending: Vec<String> = crate::coordinator::Coordinator::pending_init_names()
        .into_iter()
        .map(hive_types::Ident::into_string)
        .collect();
    crate::topology::reconcile(&agent_names, &pending)
        .with_context(|| format!("reconcile {}", crate::topology::topology_path().display()))?;

    // Refresh /var/lib/hyperhive/run/agent-sockets.json — drives the
    // gateway's unix-socket upstreams. Every agent binds a unix socket
    // (`HIVE_WEB_SOCKET`); the gateway routes via this map, falling back
    // to a computed TCP loopback port (`lifecycle::agent_web_port`) only
    // while an agent's socket marker is absent. Best-effort + non-fatal.
    // See `docs/networking/gateway.md::Per-agent unix-socket upstream`.
    if let Err(e) = crate::agent_sockets::write(&agent_names) {
        tracing::warn!(error = ?e, "agent_sockets::write failed (non-fatal)");
    }

    // Refresh /var/lib/hive-gateway/conf/agents.conf — the nginx include
    // file the gateway reads at runtime. c0re then triggers a reload (or
    // start) of the host's nginx via hive-priv, since c0re is unprivileged.
    // Same best-effort + non-fatal shape.
    if let Err(e) = crate::gateway_nginx::write(&agent_names).await {
        tracing::warn!(error = ?e, "gateway_nginx::write failed (non-fatal)");
    }

    if initial {
        git(&dir, &["init", "--initial-branch=main"]).await?;
    }
    // Stage flake.nix *before* running nix flake lock. When meta is
    // a git repo, nix treats it as a `git+file://` self-reference;
    // its dirty-tree fetcher includes index entries (tracked +
    // staged) but skips untracked files, so without the stage step
    // an untracked flake.nix surfaces as "source tree does not
    // contain '/flake.nix'". Lock then commit once with both
    // flake.nix and flake.lock — single commit per change.
    git(&dir, &["add", "flake.nix"]).await?;
    // Stage every embedded CA file we wrote or removed (trust bundle +
    // stale peer CAs). `git add <path>` stages a deletion when the path is
    // tracked and now gone; best-effort so the never-tracked-and-absent case
    // (pathspec mismatch) is a harmless no-op.
    for name in &ca_touched {
        let _ = git(&dir, &["add", "--", name]).await;
    }
    stage_generated_meta_files(&dir).await?;
    nix(&dir, &["flake", "lock"]).await?;
    if std::path::Path::new(&dir).join("flake.lock").exists() {
        git(&dir, &["add", "flake.lock"]).await?;
    }
    // Build the commit message from what's actually staged so it
    // reflects reality — and skip the commit entirely when nothing
    // changed (avoids "nothing to commit" errors on redundant syncs).
    let staged = git_staged_names(&dir).await?;
    if staged.is_empty() {
        return Ok(());
    }
    let msg = if initial {
        format!("seed meta from {} agent(s)", agents.len())
    } else {
        // Compose a message that names every file that actually changed,
        // mapping the on-disk filename to a short human label.
        let labels: Vec<&str> = staged
            .iter()
            .filter_map(|f| match f.as_str() {
                "flake.nix" => Some("flake"),
                "flake.lock" => Some("lock"),
                "hive-ca.pem" => Some("hive-ca"),
                // Only ever a REMOVAL now; see `is_embedded_ca_name`.
                f if f.starts_with("peer-ca-") && has_pem_ext(f) => Some("peer-ca"),
                "topology.json" => Some("topology"),
                _ => None,
            })
            .collect();
        if labels.is_empty() {
            "meta: update".to_owned()
        } else {
            format!("meta: update {}", labels.join(", "))
        }
    };
    git_commit(&dir, &msg).await?;
    Ok(())
}

/// One-shot used by the manual-rebuild path: relock just one
/// agent's input and commit the lock change if any. Single-phase
/// (no separate finalize) because rebuild has no failure-revert
/// semantics — it always wants the latest main.
pub async fn lock_update_for_rebuild(name: &str) -> Result<()> {
    let _guard = META_LOCK.lock().await;
    let dir = crate::paths::meta_root();
    let input = format!("agent-{name}");
    // Re-lock against `applied/<n>/main` explicitly, so the lock is the
    // exact applied head whatever the declared input url says.
    let applied = applied_override_url(&crate::paths::applied_dir(name));
    nix(
        &dir,
        &[
            "flake",
            "update",
            &input,
            "--override-input",
            &input,
            &applied,
        ],
    )
    .await?;
    if !paths_dirty(&dir, &["flake.lock"]).await? {
        return Ok(());
    }
    git(&dir, &["add", "flake.lock"]).await?;
    git_commit_paths(
        &dir,
        &format!("rebuild {name}: lock update"),
        &["flake.lock"],
    )
    .await
}

/// `--override-input` URL re-locking an agent's config input against its
/// LOCAL applied mirror (`git+file://<applied_dir>`, current `main` head),
/// the same url the meta flake declares. The deploy + rebuild paths re-lock
/// from here: the applied tree was already fast-forwarded to the reviewed
/// head, so this builds exactly that config with no forge access. No
/// `?rev` — `main` head is the reviewed head at deploy time. Pure so it's
/// unit-testable.
fn applied_override_url(applied_dir: &Path) -> String {
    format!("git+file://{}", applied_dir.display())
}

/// Update one or more named inputs in the meta flake and commit
/// the resulting lock change with a single combined message.
/// Used by the dashboard's "update meta inputs" form so the
/// operator can bulk-bump `hyperhive` + selected agents in one
/// shot. Each input name is passed verbatim to
/// Run `nix flake update [inputs...]` on the meta flake and commit the
/// resulting lock changes. When `inputs` is empty, updates ALL inputs
/// (bare `nix flake update`). The caller is responsible for picking
/// real input keys (e.g. via `inputs_view()` snapshotted from the lock
/// file) when targeting specific inputs.
pub async fn lock_update(inputs: &[String]) -> Result<()> {
    let _guard = META_LOCK.lock().await;
    let dir = crate::paths::meta_root();
    let mut args: Vec<&str> = vec!["flake", "update"];
    for i in inputs {
        args.push(i.as_str());
    }
    nix(&dir, &args).await?;
    if !paths_dirty(&dir, &["flake.lock"]).await? {
        return Ok(());
    }
    git(&dir, &["add", "flake.lock"]).await?;
    let msg = if inputs.is_empty() {
        "lock update: all inputs".to_string()
    } else if inputs.len() == 1 {
        format!("lock update: {}", inputs[0])
    } else {
        format!("lock update: {}", inputs.join(", "))
    };
    git_commit_paths(&dir, &msg, &["flake.lock"]).await
}

/// One-shot used by the auto-update path: pin the latest hyperhive
/// rev, commit if the lock changed. Cheaper than `sync_agents`
/// because the per-agent inputs aren't touched.
pub async fn lock_update_hyperhive() -> Result<()> {
    let _guard = META_LOCK.lock().await;
    let dir = crate::paths::meta_root();
    nix(&dir, &["flake", "update", "hyperhive"]).await?;
    if !paths_dirty(&dir, &["flake.lock"]).await? {
        return Ok(());
    }
    git(&dir, &["add", "flake.lock"]).await?;
    git_commit_paths(&dir, "bump hyperhive", &["flake.lock"]).await
}

/// Delete the meta repo's permission files and commit the removal under
/// `META_LOCK`. Their grants live in the swarm's `agent-permissions` bucket
/// once `workers::permissions_import` has sent them.
///
/// # Errors
///
/// Removing a file or a git stage/commit step fails.
pub async fn remove_permission_files() -> Result<()> {
    let _guard = META_LOCK.lock().await;
    let dir = crate::paths::meta_root();
    let mut removed = Vec::new();
    for file in crate::workers::permissions_import::META_FILES {
        match std::fs::remove_file(dir.join(file)) {
            Ok(()) => {}
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
            Err(e) => return Err(e).with_context(|| format!("removing {file}")),
        }
        git(&dir, &["rm", "--cached", "--ignore-unmatch", "-q", file]).await?;
        // Only a tracked file has a removal to commit; naming an untracked
        // one makes `git commit -- <path>` fail.
        if paths_dirty(&dir, &[file]).await? {
            removed.push(*file);
        }
    }
    if !removed.is_empty() {
        git_commit_paths(
            &dir,
            "permissions moved to the swarm's agent-permissions",
            &removed,
        )
        .await?;
    }
    Ok(())
}

/// [`render_flake`] with its inputs taken from `hive`, as [`sync_agents`]
/// renders it.
fn render_hive_flake(hive: &HiveEnv, agents: &[AgentSpec]) -> String {
    render_flake(
        &hive.hyperhive_flake,
        &hive.hyperhive_docs_flake,
        &hive.nixpkgs_flake,
        hive.claude_code_path.as_deref(),
        hive.dashboard_port,
        agents,
    )
}

fn render_flake(
    hyperhive_flake: &str,
    docs_flake: &str,
    nixpkgs_flake: &str,
    claude_code_path: Option<&str>,
    dashboard_port: u16,
    agents: &[AgentSpec],
) -> String {
    render_flake_with_lookup(
        hyperhive_flake,
        docs_flake,
        nixpkgs_flake,
        claude_code_path,
        dashboard_port,
        agents,
        agent_canonical_inputs,
    )
}

/// Canonical inputs meta knows how to dedup. An agent that declares one
/// of these as a top-level input in its own `flake.nix` will get a
/// `follows = "<name>"` line emitted in meta — collapsing the
/// otherwise-separate-but-identical `nixpkgs_N` nodes into a single
/// meta-level reference.
const CANONICAL_INPUTS: &[&str] = &["nixpkgs"];

/// Env vars hive-c0re forwards from its own systemd unit env into every
/// sub-agent's harness service env. Each entry is `(env_var_name,
/// host_value)`. Empty / unset vars are filtered out so absent options
/// don't render no-op `FOO = ""` lines into the meta flake.
///
/// Returns owned `String` values so the result is `'static`-friendly +
/// trivial to stub from tests (which build their own slice instead of
/// touching process-wide env).
const FORWARDED_VARS: &[&str] = &[
    // Where the swarm queue is and where its tokens are minted, as an agent
    // container reaches them. Both are addresses the *host* computes — the
    // queue's from the bridge IP an agent routes over, never a loopback
    // address, which inside a container is the agent itself.
    "HIVE_AGENT_NATS_URL",
    "HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
    // Where the swarm secret store listens. The hive's own `BAO_ADDR` value,
    // forwarded under a separate name so the container never inherits the
    // hive's store environment wholesale — the address is the only part of it
    // an agent may have, the certificate beside it being the hive's own.
    "HIVE_AGENT_BAO_ADDR",
    // Where an agent's `swarm-logs` queries, computed by the host for the
    // same reason as the two above: it is a gateway address, and a container
    // cannot derive it from anything it holds.
    "HIVE_AGENT_LOGS_QUERY_URL",
    // The metrics store's origin, for `swarm-metrics`, for the same reason.
    "HIVE_AGENT_METRICS_URL",
    "HIVE_FORGE_URL",
    "HIVE_FORGE_PUBLIC_URL",
    "HIVE_MATRIX_URL",
    "HYPERHIVE_SWARM_NAME",
];

/// Map of forwarded env var -> the agent option carrying the same value.
///
/// Both exist because they're consumed at different times: the option is baked
/// into scripts at build time (forge-avatar-sync bakes `FORGE_URL` from it), the env
/// var is read at runtime. Setting only one leaves the other on its default,
/// which is how a hive ends up with two disagreeing answers for one value.
///
/// The swarm display name is here for that reason: `otel.nix` reads the option
/// at eval time for the `swarm=` resource label, and with the env var alone
/// that label evaluates to `unknown` while the process env holds the right
/// name. Half-wiring this map is a value that evaluates fine and is silently
/// wrong.
const FORWARDED_VAR_OPTIONS: &[(&str, &str)] = &[
    // Read at build time to decide whether the harness unit imports the hive's
    // queue credential at all — see `nix/agent-modules/queue.nix`. Both halves
    // wired for this map's own reason: an agent whose option says no queue and
    // whose env says otherwise imports no credential for `swarm-logs`.
    (
        "HIVE_AGENT_NATS_URL",
        "services.hyperhive.agent.queue.natsUrl",
    ),
    (
        "HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
        "services.hyperhive.agent.queue.tokenEndpoint",
    ),
    // Read at build time to decide whether the in-container identity check is
    // generated at all — see `nix/agent-modules/bao.nix`. Both halves wired
    // for this map's own reason: the unit bakes the address into its own
    // environment, so an agent whose option says no store and whose env says
    // otherwise would have a check that is simply not there.
    ("HIVE_AGENT_BAO_ADDR", "services.hyperhive.agent.bao.addr"),
    // Read at build time to decide whether `swarm-logs` is installed at all —
    // see `nix/agent-modules/logs.nix`. Both halves wired for this map's own
    // reason: an agent whose option says no log store and whose env says
    // otherwise has a binary that is not on PATH and coordinates that claim
    // it is.
    (
        "HIVE_AGENT_LOGS_QUERY_URL",
        "services.hyperhive.agent.logs.queryUrl",
    ),
    // The same for `swarm-metrics`: see `nix/agent-modules/metrics.nix`.
    (
        "HIVE_AGENT_METRICS_URL",
        "services.hyperhive.agent.metrics.url",
    ),
    ("HIVE_FORGE_URL", "services.hyperhive.agent.forge.url"),
    ("HIVE_MATRIX_URL", "services.hyperhive.agent.matrix.url"),
    ("HYPERHIVE_SWARM_NAME", "services.hyperhive.agent.swarmName"),
];

/// Render the forwarded-var option assignments for one agent's module block.
///
/// Split out of `render_flake` so it can be tested without touching process
/// env: the render-level tests have to `set_var`, which forces them to
/// serialise on the crate's `test_env` lock (a `#[cfg(test)]` module, so this
/// deliberately isn't an intra-doc link). A pure function over the
/// already-collected pairs has no such hazard.
///
/// A var that isn't present emits nothing rather than a guess. For an optional
/// service that is the whole point — the agent option defaults to `null`,
/// meaning "not configured", and the units that would use it aren't generated.
/// For a service the hive cannot run without, silence would instead produce a
/// fleet of agents quietly missing an integration, so those are checked by
/// [`require_service_urls`] before this is called.
fn push_forwarded_var_options(out: &mut String, vars: &[(&'static str, String)]) {
    use std::fmt::Write as _;
    for (var, val) in vars {
        let Some((_, option)) = FORWARDED_VAR_OPTIONS.iter().find(|(name, _)| name == var) else {
            continue;
        };
        let escaped = val.replace('\\', "\\\\").replace('"', "\\\"");
        let _ = writeln!(out, "              {option} = \"{escaped}\";");
    }
}

/// Service URLs a running hive must always supply, checked before rendering.
///
/// The forge is not optional on a real hive: the hyperhive NixOS module sets
/// `HIVE_FORGE_URL` unconditionally, so its absence means this daemon was
/// started outside its unit. The agent option is nullable — `null`
/// legitimately means "no forge" when the modules are evaluated on their own —
/// which is exactly why the hive has to assert its own requirement here rather
/// than leaning on the module to reject the empty case.
const REQUIRED_SERVICE_URL_VARS: &[&str] = &["HIVE_FORGE_URL"];

/// Fails unless every [`REQUIRED_SERVICE_URL_VARS`] entry is present in
/// `vars`.
///
/// Pure over the already-collected pairs so it can be tested without touching
/// process env (the same reason [`push_forwarded_var_options`] is split out).
///
/// Checked by `sync_agents` — the point at which the hive commits a rendered
/// flake to disk — rather than inside the renderer. Rendering is a pure string
/// operation that many tests exercise directly; making *it* env-dependent
/// would mean every one of those tests either sets a process-wide var (the
/// parallel-test race this module already avoids) or fails for reasons that
/// have nothing to do with what it asserts.
///
/// # Errors
///
/// When a required var is missing. The hyperhive NixOS module sets it
/// unconditionally, so this means the daemon is running outside its unit;
/// refusing to
/// write the flake beats writing one whose agents would silently all lack a
/// forge.
fn require_service_urls(vars: &[(&'static str, String)]) -> Result<()> {
    for required in REQUIRED_SERVICE_URL_VARS {
        if !vars.iter().any(|(name, _)| name == required) {
            anyhow::bail!(
                "{required} is unset — the hyperhive NixOS module sets it unconditionally on \
                 the hive-c0re unit, so this process was started outside that unit. Refusing \
                 to write a meta flake whose agents would every one of them have no forge \
                 configured."
            );
        }
    }
    Ok(())
}

fn forwarded_env_vars() -> Vec<(&'static str, String)> {
    FORWARDED_VARS
        .iter()
        .filter_map(|&name| {
            std::env::var(name)
                .ok()
                .filter(|v| !v.is_empty())
                .map(|v| (name, v))
        })
        .collect()
}

/// Filename the hive's own trust anchors are embedded under at the
/// meta-flake root — the only entry of the embedded-CA list
/// `render_flake` references in `security.pki.certificateFiles` (see
/// `embedded_ca_files`).
const HIVE_CA_FILE: &str = "hive-ca.pem";

/// Host path of the hive's TLS trust anchors, when self-signed TLS is
/// active. The hyperhive TLS module sets `HIVE_TLS_CA_PATH` in hive-c0re's env
/// (to `<deploy.hive-controller.tls.stateDir>/trust-bundle.pem`) whenever the
/// gateway serves a leaf issued by the secret store's `pki` mount. The
/// file holds that mount's root, the one anchor every hive's gateway leaf
/// chains to, and is copied verbatim, which `security.pki.certificateFiles`
/// accepts; nothing here parses it. Returns `Some(path)` only when the var
/// is set AND the cert exists on disk — so render + write stay consistent
/// (we never emit a `certificateFiles` reference to a file we didn't
/// embed). Only the public cert is ever read here.
fn hive_ca_source() -> Option<String> {
    let path = std::env::var("HIVE_TLS_CA_PATH").ok()?;
    if path.is_empty() || !std::path::Path::new(&path).is_file() {
        return None;
    }
    Some(path)
}

/// Hive-wide OTEL config injected into every agent's build, read off
/// hive-c0re's own unit env (set from `services.hyperhive.otel.*`,
/// `services.hyperhive.swarm.hiveOtel.*` and
/// `services.hyperhive.deploy.hive-controller.otel.*`). A present, non-empty
/// `HYPERHIVE_OTEL_ENDPOINT` is the enable signal — the host module
/// asserts the endpoint is set whenever `otel.enable` is true, so
/// "endpoint present" == "OTEL on". The optional fields map to the
/// matching host options and are only carried when set.
struct OtelConfig {
    endpoint: String,
    protocol: String,
    extra_resource_attributes: Option<String>,
}

/// Read the hive-wide OTEL config from env, or `None` when OTEL is off.
/// Mirrors `hive_ca_source` — host state surfaced to the meta renderer
/// so it can bake build-time `services.hyperhive.agent.otel.*` config into each agent
/// (the per-agent options the harness modules consume). Returns `None`
/// when the endpoint signal is absent so the renderer emits no
/// `services.hyperhive.agent.otel.*` lines and agents keep the disabled default.
fn otel_config() -> Option<OtelConfig> {
    let endpoint = std::env::var("HYPERHIVE_OTEL_ENDPOINT")
        .ok()
        .filter(|v| !v.is_empty())?;
    let protocol = std::env::var("HYPERHIVE_OTEL_PROTOCOL")
        .ok()
        .filter(|v| !v.is_empty())
        .unwrap_or_else(|| "http/protobuf".to_owned());
    let extra_resource_attributes = std::env::var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES")
        .ok()
        .filter(|v| !v.is_empty());
    Some(OtelConfig {
        endpoint,
        protocol,
        extra_resource_attributes,
    })
}

/// The ordered set of CA certs embedded next to the meta flake, as
/// `(filename, host_source_path)` — now just the hive's own trust
/// anchors as `hive-ca.pem`, when self-signed TLS is active.
///
/// That single file carries the store's `pki` root (the hyperhive TLS
/// module writes it into the trust bundle), so every hive's gateway leaf
/// validates from it and nothing is embedded per peer.
///
/// `render_flake` emits exactly these filenames into
/// `security.pki.certificateFiles` and `sync_agents` materialises them,
/// so the rendered reference and the embedded files always agree.
fn embedded_ca_files() -> Vec<(String, String)> {
    let mut out = Vec::new();
    if let Some(p) = hive_ca_source() {
        out.push((HIVE_CA_FILE.to_owned(), p));
    }
    out
}

/// Write each desired embedded CA file next to `flake.nix` and remove
/// any stale one (self-signed TLS turned off, or a leftover peer CA),
/// so the flake never references a file we didn't write. Returns every
/// filename written or removed, for the caller to stage. The public CA
/// certs only; no private key is ever embedded.
fn materialise_ca_files(dir: &Path, ca_files: &[(String, String)]) -> Result<Vec<String>> {
    let desired: std::collections::HashSet<&str> =
        ca_files.iter().map(|(n, _)| n.as_str()).collect();
    let mut touched: Vec<String> = Vec::new();
    if let Ok(entries) = std::fs::read_dir(dir) {
        for e in entries.flatten() {
            let fname = e.file_name();
            let Some(name) = fname.to_str() else { continue };
            if is_embedded_ca_name(name) && !desired.contains(name) {
                let _ = std::fs::remove_file(dir.join(name));
                touched.push(name.to_owned());
            }
        }
    }
    for (name, content) in ca_files {
        let path = dir.join(name);
        std::fs::write(&path, content).with_context(|| format!("write {}", path.display()))?;
        touched.push(name.clone());
    }
    Ok(touched)
}

/// True for a filename the embedded-CA machinery owns. Lets
/// `sync_agents` find stale CA files to clean up (a CA dropped from
/// config) without touching unrelated meta files.
///
/// ⚠️ Still matches `peer-ca-<N>.pem`, which `embedded_ca_files` no
/// longer produces — deliberately. Cleanup is driven by
/// "recognised but not desired", so this arm is exactly what removes a
/// per-peer CA file still sitting at a meta root. Drop it and those files
/// are orphaned at every meta root forever, referenced by nothing and
/// cleaned by nobody.
fn is_embedded_ca_name(name: &str) -> bool {
    name == HIVE_CA_FILE || (name.starts_with("peer-ca-") && has_pem_ext(name))
}

/// True when `name` ends in a `.pem` extension (case-insensitive). Split
/// out so the embedded-CA filename checks share one spelling and dodge
/// clippy's case-sensitive-extension lint.
fn has_pem_ext(name: &str) -> bool {
    Path::new(name)
        .extension()
        .is_some_and(|ext| ext.eq_ignore_ascii_case("pem"))
}

/// Embedded-CA state for the meta repo: `(desired_files, changed)`.
/// `desired_files` is `(filename, contents)` for every CA that should sit
/// next to flake.nix. `changed` is true when the on-disk set differs in
/// any way — the file's contents changed, a CA appeared, or a
/// previously-embedded CA (`hive-ca.pem` / a legacy `peer-ca-*.pem`) is
/// no longer wanted (stale, to be removed). Drives both the re-commit
/// decision and the materialise/cleanup in `sync_agents`, so a CA
/// rotation re-commits even when the flake itself is byte-identical.
fn ca_embed_state(dir: &std::path::Path) -> (Vec<(String, String)>, bool) {
    let desired: Vec<(String, String)> = embedded_ca_files()
        .into_iter()
        .filter_map(|(name, path)| std::fs::read_to_string(&path).ok().map(|c| (name, c)))
        .collect();
    let desired_names: std::collections::HashSet<&str> =
        desired.iter().map(|(n, _)| n.as_str()).collect();

    let mut changed = desired.iter().any(|(name, content)| {
        std::fs::read_to_string(dir.join(name)).unwrap_or_default() != *content
    });

    // A previously-embedded CA file no longer wanted → stale (removal is
    // a change even when every desired file already matches on disk).
    if !changed && let Ok(entries) = std::fs::read_dir(dir) {
        changed = entries.flatten().any(|e| {
            e.file_name()
                .to_str()
                .is_some_and(|name| is_embedded_ca_name(name) && !desired_names.contains(name))
        });
    }
    (desired, changed)
}

/// Read an agent's applied `flake.lock` and return the subset of
/// `CANONICAL_INPUTS` it declares as direct (root-level) inputs.
/// Returns an empty vec when the lock is missing or unparsable —
/// safe degradation, the worst case is no dedup for that agent.
fn agent_canonical_inputs(name: &str) -> Vec<&'static str> {
    let path = crate::paths::applied_dir(name).join("flake.lock");
    let Ok(raw) = std::fs::read_to_string(&path) else {
        return Vec::new();
    };
    let Ok(json) = serde_json::from_str::<serde_json::Value>(&raw) else {
        return Vec::new();
    };
    let Some(nodes) = json.get("nodes").and_then(|v| v.as_object()) else {
        return Vec::new();
    };
    let Some(root_name) = json.get("root").and_then(|v| v.as_str()) else {
        return Vec::new();
    };
    let Some(root_inputs) = nodes
        .get(root_name)
        .and_then(|n| n.get("inputs"))
        .and_then(|v| v.as_object())
    else {
        return Vec::new();
    };
    CANONICAL_INPUTS
        .iter()
        .copied()
        .filter(|canon| root_inputs.contains_key(*canon))
        .collect()
}

/// Inner render helper accepting a lookup fn so tests can stub the
/// agent flake-lock introspection.
#[allow(
    clippy::too_many_lines,
    reason = "templated string-builder for the meta flake — the length is one \
              contiguous fmt block, splitting it would just hide the shape"
)]
fn render_flake_with_lookup<F>(
    hyperhive_flake: &str,
    docs_flake: &str,
    nixpkgs_flake: &str,
    claude_code_path: Option<&str>,
    dashboard_port: u16,
    agents: &[AgentSpec],
    lookup: F,
) -> String
where
    F: Fn(&str) -> Vec<&'static str>,
{
    use std::fmt::Write as _;
    let mut out = String::new();
    out.push_str("{\n  description = \"hyperhive deployed agents\";\n  inputs = {\n");
    // `nixpkgs` is a top-level meta input with an explicit store-path URL.
    // `hyperhive` then follows it via `hyperhive.inputs.nixpkgs.follows`.
    // This cascades through to every agent because
    // `agent-<n>.inputs.nixpkgs.follows = "nixpkgs"` resolves to the same
    // top-level node.
    //
    // Why an explicit `path:` URL instead of
    // `nixpkgs.follows = "hyperhive/nixpkgs"`:
    // meta points to hyperhive's *store path* as its flake input, so nix
    // reads hyperhive's own pinned lock when evaluating that input — the
    // host-level `follows` the operator set never propagates. Injecting the
    // evaluated `pkgs.path` directly at nix-module evaluation time is the
    // only reliable way to honour the host's channel choice.
    //
    // Fallback (flake arg empty): legacy `follows` wiring — used when
    // hive-c0re is not built with this option wired up.
    if nixpkgs_flake.is_empty() {
        // Legacy path: meta defers to hyperhive's own lock.
        let _ = writeln!(out, "    hyperhive.url = \"{hyperhive_flake}\";");
        out.push_str("    nixpkgs.follows = \"hyperhive/nixpkgs\";\n");
    } else {
        let _ = writeln!(out, "    nixpkgs.url = \"{nixpkgs_flake}\";");
        let _ = writeln!(out, "    hyperhive.url = \"{hyperhive_flake}\";");
        out.push_str("    hyperhive.inputs.nixpkgs.follows = \"nixpkgs\";\n");
    }
    // Narrow `docs/` source as its own input so a doc edit only
    // re-locks THIS input instead of re-hashing the whole `hyperhive`
    // source. Threaded to each agent below as `services.hyperhive.agent.docs.source`.
    // Empty = hive-c0re not built with the option wired up (legacy);
    // agents then keep the harness default (`services.hyperhive.agent.packages.reference-docs`).
    if !docs_flake.is_empty() {
        // `flake = false`: the docs/ tree is a plain source (no flake.nix),
        // so nix must treat it as raw source, not evaluate it as a flake.
        let _ = writeln!(out, "    hyperhive-docs.url = \"{docs_flake}\";");
        out.push_str("    hyperhive-docs.flake = false;\n");
    }
    // Each agent's config input is its LOCAL applied mirror
    // (`git+file://<applied>`), so the meta flake resolves entirely from
    // on-disk state and boot never depends on the forge being reachable.
    // The forge `agent-configs/<name>` repos stay the review/audit surface
    // (config PRs land there) but are not the flake's build input. deploy +
    // rebuild re-lock this input to `applied/<name>`'s current `main` head.
    for spec in agents {
        // An archetype agent has no config repo, so no input: `mkAgent`
        // takes its module from the archetype.
        if spec.archetype.is_some() {
            continue;
        }
        let _ = writeln!(
            out,
            "    agent-{name}.url = \"git+file://{applied}\";",
            name = spec.name,
            applied = crate::paths::applied_dir(&spec.name).display(),
        );
        // For each canonical input the agent declares in its own
        // `flake.nix` (detected by reading its applied `flake.lock`),
        // emit `inputs.agent-<name>.inputs.<canon>.follows = "<canon>"`.
        // Collapses otherwise-separate-but-identical nixpkgs nodes
        // (root + every agent's own nixpkgs) into one. Skipped
        // silently for agents that don't declare the input — emitting
        // follows on a non-existent input would error at
        // `nix flake lock` time.
        for canon in lookup(&spec.name) {
            let _ = writeln!(
                out,
                "    agent-{}.inputs.{canon}.follows = \"{canon}\";",
                spec.name,
            );
        }
    }
    out.push_str("  };\n  outputs =\n    { self, hyperhive, ... }@inputs:\n    let\n");
    let _ = writeln!(
        out,
        "      dashboardPort = {dashboard_port};\n      mkAgent = {{ name, port, archetype ? null }}:"
    );
    out.push_str(
        r#"        let
          base = hyperhive.nixosConfigurations.agent-base;
          # Every flake an archetype can come from, by its source name.
          archetypeSources = { inherit hyperhive; };
          agentModule =
            if archetype == null then
              inputs."agent-${name}".nixosModules.default
            else
              (archetypeSources.${archetype.source}
                or (throw "agent ${name}: archetype source ${archetype.source} is not an input of this hive's meta flake")
              ).hyperhiveArchetypes.${archetype.name}.module
                or (throw "agent ${name}: this hive's ${archetype.source} input has no archetype ${archetype.name}; update that input");
          service = "hive-agent";
        in
        base.extendModules {
          modules = [
            agentModule
            {
"#,
    );
    // Point the in-container docs dir (`$HIVE_DOCS_DIR`) at the narrow
    // `hyperhive-docs` input instead of the harness default
    // (`services.hyperhive.agent.packages.reference-docs`, built from the now-docs-stripped source).
    // `inputs."hyperhive-docs"` is reachable via the outputs `@inputs`
    // capture. Emitted only when the input exists (docs_flake non-empty).
    if !docs_flake.is_empty() {
        out.push_str("              services.hyperhive.agent.docs.source = inputs.\"hyperhive-docs\".outPath;\n");
    }
    // The `claude-code` agents run, as a bare store path rather than a
    // flake input: containers share the host's `/nix/store`, so the
    // binary is already there with its closure and has nothing to
    // travel. A string literal is also the only shape that evaluates —
    // `lib.types.package` on a bare path runs `builtins.storePath`,
    // which pure eval rejects. The agent module puts its `bin/` on the
    // harness PATH; the host module holds the gc root, since a path
    // spelled out here is text and references nothing.
    // `None` = no override; agents keep their own nixpkgs' `claude-code`.
    if let Some(path) = claude_code_path {
        let _ = writeln!(
            out,
            "              services.hyperhive.agent.claudeCodePath = \"{path}\";"
        );
    }
    // CA trust: embed the hive's trust bundle so each agent validates
    // the store's `pki` root at build time (one file; see
    // `hive_ca_source`). `certificateFiles` is build-time, so the certs
    // travel with the flake source: `sync_agents` writes `./hive-ca.pem`
    // next to flake.nix and stages it. Only public CA certs are embedded;
    // no private key ever leaves the host. The filename list matches
    // `sync_agents` exactly (both derive it from `embedded_ca_files`), so
    // we never reference a file we didn't embed; emitted only when the
    // list is non-empty.
    let ca_refs: Vec<String> = embedded_ca_files()
        .into_iter()
        .map(|(name, _)| format!("./{name}"))
        .collect();
    if !ca_refs.is_empty() {
        let _ = writeln!(
            out,
            "              security.pki.certificateFiles = [ {} ];",
            ca_refs.join(" ")
        );
    }
    // Hive-wide OTEL stats export (`services.hyperhive.otel.*`): inject the
    // build-time `services.hyperhive.agent.otel.*` config the harness modules consume (its
    // otelEnv + otelExecStart wrapper + LoadCredential). Host-driven, so
    // the same config lands on every agent; emitted only when enabled.
    // Mirrors the CA-cert injection above — host state -> build-time agent
    // module config.
    if let Some(otel) = otel_config() {
        let esc = |s: &str| s.replace('\\', "\\\\").replace('"', "\\\"");
        out.push_str("              services.hyperhive.agent.otel.enable = true;\n");
        let _ = writeln!(
            out,
            "              services.hyperhive.agent.otel.endpoint = \"{}\";",
            esc(&otel.endpoint)
        );
        let _ = writeln!(
            out,
            "              services.hyperhive.agent.otel.protocol = \"{}\";",
            esc(&otel.protocol)
        );
        if let Some(attrs) = &otel.extra_resource_attributes {
            let _ = writeln!(
                out,
                "              services.hyperhive.agent.otel.extraResourceAttributes = \"{}\";",
                esc(attrs)
            );
        }
    }
    // Agent-facing service URLs (`services.hyperhive.agent.forge.url`, `services.hyperhive.agent.matrix.url`):
    // emit the host's real values as build-time agent config, the same
    // host-state -> agent-module shape as the otel block above.
    //
    // These options exist already, but until now *nothing set them*, so every
    // agent silently fell back to their `localhost:<port>` defaults while the
    // true value reached the container only as an env var (below). Two sources
    // of truth that disagree, with the winner decided by which code path
    // happens to read which one --- the option default is baked into scripts
    // at build time, the env var is read at runtime.
    //
    // Rendering them here makes the option the single source, which is the
    // precondition for removing those defaults: a loopback address is only
    // ever correct when the callee shares the caller's netns, and the forge
    // and homeserver are moving to swarm level, possibly on other hosts.
    //
    // Absent vars emit nothing rather than a guess, and the agent option
    // treats "unset" as "this service is not configured" rather than
    // substituting a loopback address — an absent integration instead of a
    // misdirected one.
    //
    // That is the right default for an optional service and the wrong one for
    // the forge, which a running hive always has — so `sync_agents` rejects a
    // missing `HIVE_FORGE_URL` before it writes anything (`require_service_urls`).
    // The check lives there rather than here because rendering is a pure string
    // operation the tests exercise directly.
    push_forwarded_var_options(&mut out, &forwarded_env_vars());
    // GitHub integration is on by default in every agent
    // (`services.hyperhive.agent.github.enable`); the host turns it off swarm-wide via
    // `services.hyperhive.swarm.github.enable = false`, surfaced here as the
    // `HYPERHIVE_GITHUB_DISABLED` env on hive-c0re's unit. Only the OFF
    // override is propagated — the enabled default needs no per-agent line.
    if std::env::var_os("HYPERHIVE_GITHUB_DISABLED").is_some() {
        out.push_str("              services.hyperhive.agent.github.enable = false;\n");
    }
    out.push_str(
        r#"              # The harness service inside the container runs as a
              # non-root unix user named after the agent (`damocles`,
              # `iris`, `root`, …). UID auto-assigned by NixOS; the
              # per-agent override here is what makes
              # `services.hyperhive.agent.user.name` match the agent's identity
              # instead of the harness default of `"agent"`.
              services.hyperhive.agent.user.name = name;
              # Without this an agent container is called `nixos` and ships
              # every log line under that: nixpkgs sets the hostname from the
              # attr name in the `containers.<name>` submodule, and an
              # agent's system is its own `nixosSystem` off this flake, which
              # never evaluates that option. The machine name, not the
              # logical one — it is what `container.name` on our metrics and
              # nspawn itself already call this container, so logs join to
              # both without a prefix transform.
              networking.hostName = "h-${name}";
              programs.git.config.user = {
                name = name;
                email = "${name}@hyperhive.local";
              };
              # Container-wide env: every service + co-process daemon can
              # resolve the agent's durable state dir without hard-coding it.
              # `environment.variables` only writes /etc/environment (login
              # shells); `systemd.globalEnvironment` is the analogue for
              # systemd units so forge-avatar-sync /
              # hive-matrix-daemon etc. can read `$HYPERHIVE_STATE_DIR`
              # without each service having to redeclare it.
              environment.variables = {
                HIVE_LABEL = name;
                HYPERHIVE_STATE_DIR = "/agents/${name}/state";
                HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
              };
              systemd.globalEnvironment = {
                HIVE_LABEL = name;
                HYPERHIVE_STATE_DIR = "/agents/${name}/state";
                HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
"#,
    );
    // Forwarded vars (HIVE_FORGE_URL etc.) also go into globalEnvironment,
    // not just the harness service env below, so EVERY service + shell in
    // the container inherits them — crucially the bash-task runner (where
    // `hive-forge` + `git` actually run), plus the matrix daemon, forge-avatar-sync
    // and interactive shells. Scoped to the harness service alone they were
    // invisible to bash tasks: harmless in shared netns (the localhost
    // default works) but broken under isolation, where the in-cluster
    // `forge.<domain>` URL is the only reachable path.
    for (var, val) in forwarded_env_vars() {
        let escaped = val.replace('\\', "\\\\").replace('"', "\\\"");
        let _ = writeln!(out, "                {var} = \"{escaped}\";");
    }
    out.push_str(
        r"              };
              systemd.services.${service}.environment = {
                HIVE_PORT = toString port;
                HIVE_LABEL = name;
                HIVE_DASHBOARD_PORT = toString dashboardPort;
",
    );
    // Forwarded env vars — picked up from hive-c0re's own systemd unit
    // env (`services.hyperhive.*` options flow through the hyperhive
    // NixOS module into the host process). We copy whatever's set into
    // each sub-agent's harness service env so the in-container surfaces
    // (`identity.rs`, `forge_notify`) see a consistent view across the
    // whole hive. Absent host-side env (option not set) → skip emission
    // → in-container accessors fall back to None / defaults gracefully.
    //
    // - HIVE_FORGE_URL: agents poll this for Forgejo notifications.
    // - HYPERHIVE_SWARM_NAME: human display name for the swarm.
    for (var, val) in forwarded_env_vars() {
        let escaped = val.replace('\\', "\\\\").replace('"', "\\\"");
        let _ = writeln!(out, "                {var} = \"{escaped}\";");
    }
    out.push_str(
        r#"                HYPERHIVE_STATE_DIR = "/agents/${name}/state";
                HYPERHIVE_HARNESS_DIR = "/agents/${name}/harness";
              };
            }
            # Tells the harness the agent has no config repo of its own.
            (if archetype == null then { } else {
              systemd.services.${service}.environment.HIVE_ARCHETYPE = archetype.name;
            })
          ];
        };
    in
    {
      nixosConfigurations = {
"#,
    );
    for spec in agents {
        // `source` and `name` are identifiers (`archetypes::store_read`), so
        // they need no escaping.
        let archetype = spec.archetype.as_ref().map_or(String::new(), |a| {
            format!(
                " archetype = {{ source = \"{}\"; name = \"{}\"; }};",
                a.source, a.name
            )
        });
        let _ = writeln!(
            out,
            "        {} = mkAgent {{ name = \"{}\"; port = {};{archetype} }};",
            spec.name, spec.name, spec.port,
        );
    }
    out.push_str("      };\n    };\n}\n");
    out
}

/// Return the list of file names that are currently staged (index differs
/// from HEAD). On the initial commit (`HEAD` doesn't exist yet) falls back
/// to `git diff --cached --name-only HEAD` failing gracefully by using
/// `git status --porcelain` and collecting the `A ` / `M ` prefix lines.
async fn git_staged_names(dir: &Path) -> Result<Vec<String>> {
    // `--diff-filter=ACM` skips deleted entries — we only care about
    // additions and modifications for message-building purposes.
    let out = lifecycle::git_command()
        .current_dir(dir)
        .args(["diff", "--cached", "--name-only", "--diff-filter=ACM"])
        .output()
        .await
        .with_context(|| format!("git diff --cached in {}", dir.display()))?;
    if out.status.success() {
        let names = String::from_utf8_lossy(&out.stdout)
            .lines()
            .map(str::trim)
            .filter(|l| !l.is_empty())
            .map(ToOwned::to_owned)
            .collect();
        return Ok(names);
    }
    // Fallback for initial commit (no HEAD yet): parse `git status --porcelain`.
    let st = lifecycle::git_command()
        .current_dir(dir)
        .args(["status", "--porcelain"])
        .output()
        .await
        .with_context(|| format!("git status in {}", dir.display()))?;
    let names = String::from_utf8_lossy(&st.stdout)
        .lines()
        .filter(|l| l.starts_with("A ") || l.starts_with("M "))
        .filter_map(|l| l.get(3..))
        .map(ToOwned::to_owned)
        .collect();
    Ok(names)
}

async fn git(dir: &Path, args: &[&str]) -> Result<()> {
    let out = lifecycle::git_command()
        .current_dir(dir)
        .args(args)
        .output()
        .await
        .with_context(|| format!("git {} in {}", args.join(" "), dir.display()))?;
    if !out.status.success() {
        bail!(
            "git {} failed ({}): {}",
            args.join(" "),
            out.status,
            String::from_utf8_lossy(&out.stderr).trim()
        );
    }
    Ok(())
}

async fn git_commit(dir: &Path, message: &str) -> Result<()> {
    git(
        dir,
        &[
            "-c",
            &format!("user.name={GIT_NAME}"),
            "-c",
            &format!("user.email={GIT_EMAIL}"),
            "commit",
            "-m",
            message,
        ],
    )
    .await?;
    // Best-effort mirror to the bundled forge. No-op when the forge
    // isn't seeded (no core token on disk); push failures log a warn
    // but don't bubble up — a missing mirror shouldn't fail an
    // otherwise successful deploy.
    if let Err(e) = crate::forge::push_meta(dir).await {
        tracing::warn!(error = ?e, "forge: meta push after commit failed (non-fatal)");
    }
    Ok(())
}

/// Path-limited commit: commits ONLY the given paths, so unrelated
/// staged content can never be swept into someone else's commit. Every targeted
/// meta commit (limits, topology, lock bumps) goes through this;
/// only `sync_agents` uses the bare [`git_commit`], because its
/// staged set *is* its intentional commit set.
async fn git_commit_paths(dir: &Path, message: &str, paths: &[&str]) -> Result<()> {
    let name = format!("user.name={GIT_NAME}");
    let email = format!("user.email={GIT_EMAIL}");
    let mut args = vec!["-c", &name, "-c", &email, "commit", "-m", message, "--"];
    args.extend_from_slice(paths);
    git(dir, &args).await?;
    if let Err(e) = crate::forge::push_meta(dir).await {
        tracing::warn!(error = ?e, "forge: meta push after commit failed (non-fatal)");
    }
    Ok(())
}

/// True when any of `paths` differs between HEAD and the index or
/// working tree — the path-scoped replacement for whole-tree
/// `git_is_clean` / `has_staged_changes` guards, which a concurrently
/// staged deploy lock would otherwise trip.
async fn paths_dirty(dir: &Path, paths: &[&str]) -> Result<bool> {
    let mut args = vec!["diff", "--quiet", "HEAD", "--"];
    args.extend_from_slice(paths);
    let out = lifecycle::git_command()
        .current_dir(dir)
        .args(&args)
        .output()
        .await
        .with_context(|| format!("git diff --quiet in {}", dir.display()))?;
    // Exit 0 = no differences; 1 = differences; anything else (e.g.
    // no HEAD yet on a fresh repo) → treat as dirty so the commit
    // path runs and surfaces real errors loudly.
    Ok(!out.status.success())
}

/// Full argv for a `nix` invocation, prefixed with the flakes-enabling
/// experimental-features flag. `--extra-experimental-features` is
/// belt-and-suspenders for hosts that haven't set this in nix.conf: the
/// hyperhive module's deploy guide assumes flakes are already enabled,
/// but the cost of being defensive is one extra argv each call.
fn nix_argv<'a>(args: &[&'a str]) -> Vec<&'a str> {
    let mut all = vec!["--extra-experimental-features", "nix-command flakes"];
    all.extend_from_slice(args);
    all
}

/// Run `nix <args>` in `dir` (flakes enabled), capturing combined output.
/// The core of [`nix`].
async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
    Command::new("nix")
        .current_dir(dir)
        .args(nix_argv(args))
        // Kill the nix child if the caller's future is dropped (e.g. the
        // startup-migration timeout around `sync_agents` fires) rather than
        // orphaning it against an unreachable forge. No-op on normal exit.
        .kill_on_drop(true)
        .output()
        .await
        .with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))
}

/// Turn a finished nix [`Output`](std::process::Output) into `Result<()>`,
/// bailing with the trimmed stderr tail on non-zero exit.
fn nix_check(args: &[&str], out: &std::process::Output) -> Result<()> {
    if !out.status.success() {
        bail!(
            "nix {} failed ({}): {}",
            args.join(" "),
            out.status,
            String::from_utf8_lossy(&out.stderr).trim()
        );
    }
    Ok(())
}

async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
    let out = nix_output(dir, args).await?;
    nix_check(args, &out)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A path-limited commit must leave an unrelated staged file untouched.
    #[tokio::test]
    async fn path_limited_commit_leaves_unrelated_staged_file_alone() {
        let tmp = tempfile::tempdir().expect("tempdir");
        let dir = tmp.path();
        git(dir, &["init", "--initial-branch=main"])
            .await
            .expect("git init");
        std::fs::write(dir.join("resource-limits.json"), "{}").expect("write");
        std::fs::write(dir.join("flake.lock"), "v1").expect("write");
        git(dir, &["add", "-A"]).await.expect("add");
        git_commit(dir, "seed").await.expect("seed commit");
        // Something stages a new lock (uncommitted)…
        std::fs::write(dir.join("flake.lock"), "v2-staged-by-deploy").expect("write");
        git(dir, &["add", "flake.lock"]).await.expect("stage lock");
        // …and a limits change commits, path-limited.
        std::fs::write(dir.join("resource-limits.json"), r#"{"alice":[]}"#).expect("write");
        git(dir, &["add", "resource-limits.json"])
            .await
            .expect("add");
        git_commit_paths(
            dir,
            "set resource limits for alice",
            &["resource-limits.json"],
        )
        .await
        .expect("path-limited commit");
        // The limits file is committed; the staged lock is not.
        assert!(
            !paths_dirty(dir, &["resource-limits.json"])
                .await
                .expect("check"),
            "limits file must be committed"
        );
        assert!(
            paths_dirty(dir, &["flake.lock"]).await.expect("check"),
            "staged deploy lock must survive the limits commit"
        );
    }

    fn sample_spec(name: &str, port: u16) -> AgentSpec {
        AgentSpec {
            name: name.to_owned(),
            port,
            archetype: None,
        }
    }

    #[test]
    fn applied_override_url_targets_local_mirror_main_head() {
        // Deploy + rebuild re-lock against the local applied mirror's `main`
        // head (no `?rev`) — so a rebuild survives forge unreachability and
        // builds the fast-forwarded config.
        let p = Path::new("/var/lib/hyperhive/agents/iris/applied");
        let url = applied_override_url(p);
        assert_eq!(url, "git+file:///var/lib/hyperhive/agents/iris/applied");
        assert!(
            !url.contains("?rev="),
            "must lock main head, not a pinned rev: {url}"
        );
    }

    #[test]
    fn render_flake_uses_explicit_nixpkgs_url_when_provided() {
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        // nixpkgs is a top-level input with an explicit URL; hyperhive
        // follows it.
        assert!(
            out.contains("nixpkgs.url = \"path:/nix/store/aaaa-nixpkgs-source\""),
            "expected explicit nixpkgs.url:\n{out}"
        );
        assert!(
            out.contains("hyperhive.inputs.nixpkgs.follows = \"nixpkgs\""),
            "expected hyperhive.inputs.nixpkgs.follows:\n{out}"
        );
        assert!(
            !out.contains("nixpkgs.follows = \"hyperhive"),
            "old-style follows must not appear when flake args are set:\n{out}"
        );
        // the narrow docs source is its own non-flake input, and each
        // agent's docs dir resolves from it rather than services.hyperhive.agent.packages.reference-docs.
        assert!(
            out.contains("hyperhive-docs.url = \"path:/nix/store/bbbb-hyperhive-docs-source\""),
            "expected hyperhive-docs input url:\n{out}"
        );
        assert!(
            out.contains("hyperhive-docs.flake = false;"),
            "docs source is not a flake, must be flake = false:\n{out}"
        );
        assert!(
            out.contains(
                "services.hyperhive.agent.docs.source = inputs.\"hyperhive-docs\".outPath;"
            ),
            "expected per-agent docs source wired to the input:\n{out}"
        );
    }

    #[test]
    fn render_flake_omits_docs_input_when_docs_flake_empty() {
        // Legacy / not-wired-up: empty docs_flake emits no docs input and
        // leaves each agent on the harness default (services.hyperhive.agent.packages.reference-docs).
        let out = render_flake(
            "github:example/hyperhive",
            "",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        assert!(
            !out.contains("hyperhive-docs"),
            "no docs input/source when docs_flake is empty:\n{out}"
        );
    }

    #[test]
    fn render_flake_names_each_agent_container_after_the_agent() {
        let out = render_flake(
            "github:example/hyperhive",
            "",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        assert!(
            out.contains(r#"networking.hostName = "h-${name}";"#),
            "an agent container with no hostname of its own is called `nixos`, \
             and ships every log line under that:\n{out}"
        );
        // The machine name `nixos-container` and our own metrics already use,
        // so a log line joins to a metric without a prefix transform.
        assert!(
            !out.contains("networking.hostName = name;"),
            "must be the machine name, not the logical one:\n{out}"
        );
        // Inside `mkAgent`'s module block, so it applies to every agent and
        // resolves to that agent's name — not emitted once at the top level.
        let mk = out.find("mkAgent = {").expect("mkAgent block");
        let host = out
            .find(r#"networking.hostName = "h-${name}";"#)
            .expect("hostname line");
        assert!(
            host > mk,
            "hostname must be emitted inside the per-agent module block:\n{out}"
        );
    }

    #[test]
    fn render_flake_pins_claude_path_without_adding_an_input() {
        // The host-pinned claude travels as a bare store path assigned to
        // an option — deliberately NOT as a flake input. Containers share
        // the host store, so the build is already reachable; making it an
        // input would re-copy it as a reference-less `-source` and strip
        // the closure the binary actually needs.
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            Some("/nix/store/cccc-claude-code-2.1.220"),
            8000,
            &[sample_spec("alice", 9001)],
        );
        assert!(
            out.contains(
                "services.hyperhive.agent.claudeCodePath = \"/nix/store/cccc-claude-code-2.1.220\";"
            ),
            "claude path assigned as a plain string literal:\n{out}"
        );
        assert!(
            !out.contains("claude-code-2.1.220\".url"),
            "the pinned claude must not become a flake input:\n{out}"
        );
    }

    #[test]
    fn render_flake_omits_claude_path_when_unset() {
        // `None` = no host-level pin: the option is left undefined so the
        // agent module keeps its own nixpkgs' `claude-code` (and keeps it
        // in `environment.systemPackages`, which is what makes the
        // unpinned case self-contained).
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        assert!(
            !out.contains("claudeCodePath"),
            "no claude assignment when unpinned:\n{out}"
        );
    }

    #[test]
    fn render_flake_falls_back_to_follows_when_nixpkgs_flake_empty() {
        // Empty nixpkgs_flake → legacy follows behaviour (backward compat
        // for any code path that can't inject pkgs.path).
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        assert!(
            out.contains("nixpkgs.follows = \"hyperhive/nixpkgs\""),
            "expected fallback follows:\n{out}"
        );
        assert!(
            !out.contains("nixpkgs.url ="),
            "no explicit url should be emitted in fallback mode:\n{out}"
        );
    }

    #[test]
    fn render_flake_emits_follows_for_agents_declaring_nixpkgs() {
        // Stub lookup: pretend `bitburner` + `dmatrix` declare `nixpkgs`
        // at their root, while `argus` has no canonical inputs at all.
        let lookup = |name: &str| -> Vec<&'static str> {
            match name {
                "bitburner" | "dmatrix" => vec!["nixpkgs"],
                _ => vec![],
            }
        };
        let out = render_flake_with_lookup(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[
                sample_spec("argus", 9001),
                sample_spec("bitburner", 9002),
                sample_spec("dmatrix", 9003),
            ],
            lookup,
        );
        // bitburner declares nixpkgs → follows emitted.
        assert!(
            out.contains("agent-bitburner.inputs.nixpkgs.follows = \"nixpkgs\""),
            "missing bitburner nixpkgs follows:\n{out}"
        );
        // dmatrix declares nixpkgs → follows emitted.
        assert!(out.contains("agent-dmatrix.inputs.nixpkgs.follows = \"nixpkgs\""));
        // argus declares neither → no follows emitted for it. Asserting
        // ABSENCE is the important bit: emitting a follows on a
        // non-existent input errors at `nix flake lock` time.
        assert!(
            !out.contains("agent-argus.inputs.nixpkgs"),
            "argus shouldn't have nixpkgs follows:\n{out}"
        );
    }

    #[test]
    fn render_flake_builds_every_agent_from_agent_base() {
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001), sample_spec("ruth", 9002)],
        );
        assert_eq!(
            out.matches("hyperhive.nixosConfigurations.").count(),
            1,
            "one base for every agent:\n{out}"
        );
        assert!(
            out.contains("base = hyperhive.nixosConfigurations.agent-base;"),
            "{out}"
        );
        for line in [
            "alice = mkAgent { name = \"alice\"; port = 9001; };",
            "ruth = mkAgent { name = \"ruth\"; port = 9002; };",
        ] {
            assert!(out.contains(line), "missing `{line}`:\n{out}");
        }
    }

    #[test]
    fn render_flake_skips_canonical_follows_when_lookup_returns_empty() {
        let out = render_flake_with_lookup(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
            |_| Vec::new(),
        );
        // No agent-side follows when the lookup reports nothing
        // declared — protects agents whose flake.lock can't be read
        // (missing / unparsable) from being broken by a follows on a
        // non-existent input.
        assert!(
            !out.contains("agent-alice.inputs."),
            "alice shouldn't have any inputs follows:\n{out}"
        );
    }

    /// An archetype agent has no `agent-<name>` input, and `mkAgent` is
    /// handed its archetype; a custom agent beside it keeps its input.
    #[test]
    fn render_flake_builds_an_archetype_agent_from_its_archetype() {
        let android = AgentSpec {
            archetype: Some(swarm_queue_client::agent_archetype::ArchetypeRef {
                source: "hyperhive".to_owned(),
                name: "android-dev".to_owned(),
            }),
            ..sample_spec("droid", 9002)
        };
        let out = render_flake_with_lookup(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001), android],
            |_| vec!["nixpkgs"],
        );
        assert!(
            !out.contains("agent-droid"),
            "an archetype agent has no config input:\n{out}"
        );
        assert!(
            out.contains(
                "droid = mkAgent { name = \"droid\"; port = 9002; \
                 archetype = { source = \"hyperhive\"; name = \"android-dev\"; }; };"
            ),
            "{out}"
        );
        assert!(out.contains("agent-alice.url = \"git+file://"), "{out}");
        assert!(out.contains("agent-alice.inputs.nixpkgs.follows"), "{out}");
        assert!(
            out.contains("alice = mkAgent { name = \"alice\"; port = 9001; };"),
            "{out}"
        );
        assert!(
            out.contains("hyperhiveArchetypes.${archetype.name}.module"),
            "{out}"
        );
        assert!(
            out.contains("environment.HIVE_ARCHETYPE = archetype.name"),
            "{out}"
        );
    }

    #[test]
    fn render_flake_forwards_env_into_global_environment() {
        // Regression for the isolation breakage where forwarded vars
        // (HIVE_FORGE_URL etc.) landed only on the harness service env,
        // so the bash-task runner / matrix daemon / shells defaulted to
        // localhost and couldn't reach the in-cluster gateway. They must
        // also appear in `systemd.globalEnvironment`, which every unit +
        // shell in the container inherits.
        //
        // Serialised against every other env-mutating test in the crate:
        // `render_flake_sets_service_url_options_from_forwarded_env` sets this
        // same variable, and both land in the one test binary.
        let _env = crate::test_env::lock();
        // SAFETY: serialised by the guard above; restored before returning.
        unsafe {
            std::env::set_var("HIVE_FORGE_URL", "http://forge.example.test");
        }
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        unsafe {
            std::env::remove_var("HIVE_FORGE_URL");
        }
        // The var must be emitted inside the globalEnvironment block, i.e.
        // before the per-service harness env block that follows it.
        let global_at = out
            .find("systemd.globalEnvironment = {")
            .expect("globalEnvironment block must exist");
        let service_at = out
            .find("systemd.services.${service}.environment")
            .expect("harness service env block must exist");
        let forge_at = out
            .find("HIVE_FORGE_URL = \"http://forge.example.test\"")
            .expect("HIVE_FORGE_URL must be forwarded into the flake");
        assert!(
            forge_at > global_at && forge_at < service_at,
            "HIVE_FORGE_URL must land inside systemd.globalEnvironment, \
             not only the harness service env:\n{out}"
        );
    }

    /// The hive's name and domain reach an agent at runtime
    /// (`/etc/hyperhive-hive-facts`), so the rendered flake does not change with
    /// either.
    #[test]
    fn render_flake_is_identical_across_hive_names_and_domains() {
        const VARS: [&str; 2] = ["HYPERHIVE_HIVE_NAME", "HYPERHIVE_HIVE_DOMAIN"];
        let _env = crate::test_env::lock();
        let saved = VARS.map(std::env::var_os);
        let render_on = |name: &str, domain: &str| {
            // SAFETY: serialised by the guard above; restored before returning.
            unsafe {
                std::env::set_var(VARS[0], name);
                std::env::set_var(VARS[1], domain);
            }
            render_flake(
                "github:example/hyperhive",
                "path:/nix/store/bbbb-hyperhive-docs-source",
                "path:/nix/store/aaaa-nixpkgs-source",
                None,
                8000,
                &[sample_spec("alice", 9001)],
            )
        };
        let alpha = render_on("alpha-hive", "alpha-hive.example.test");
        let beta = render_on("beta-hive", "beta-hive.example.test");
        for (var, value) in VARS.iter().zip(saved) {
            // SAFETY: as above.
            unsafe {
                match value {
                    Some(v) => std::env::set_var(var, v),
                    None => std::env::remove_var(var),
                }
            }
        }
        for hive in ["alpha-hive", "beta-hive"] {
            assert!(
                !alpha.contains(hive) && !beta.contains(hive),
                "the rendered flake must not name the hive ({hive}):\n{alpha}"
            );
        }
        assert_eq!(alpha, beta, "the render must not depend on the hive");
    }

    #[test]
    fn service_url_options_render_from_forwarded_pairs() {
        // Pure over the pairs, deliberately: the render-level tests have to
        // mutate process env, which makes them race each other under the
        // parallel runner — the first version of this test did exactly that
        // and failed for that reason rather than for a real defect.
        let mut out = String::new();
        push_forwarded_var_options(
            &mut out,
            &[
                ("HIVE_FORGE_URL", "http://forge.example.test".to_string()),
                ("HIVE_MATRIX_URL", "http://matrix.example.test".to_string()),
                // Mapped too: `otel.nix` reads the option at eval time.
                ("HYPERHIVE_SWARM_NAME", "constellat1on".to_string()),
                // Forwarded but genuinely unmapped — proves the map is a
                // filter, not a pass-through.
                (
                    "HIVE_FORGE_PUBLIC_URL",
                    "https://forge.example.test".to_string(),
                ),
            ],
        );
        assert_eq!(
            out,
            "              services.hyperhive.agent.forge.url = \"http://forge.example.test\";\n\
             \x20             services.hyperhive.agent.matrix.url = \"http://matrix.example.test\";\n\
             \x20             services.hyperhive.agent.swarmName = \"constellat1on\";\n",
            "expected exactly the mapped options, in input order, indented for the module block"
        );
    }

    /// The queue coordinates travel as a pair and are wired through both
    /// halves of the forwarded-var machinery: the env var the harness reads at
    /// runtime and the option the harness *unit* is built from. An agent that
    /// got only the env var would declare no credential and then report a
    /// partial config — visible, but a rebuild away from working.
    #[test]
    fn queue_coordinates_are_forwarded_as_both_env_and_option() {
        for var in ["HIVE_AGENT_NATS_URL", "HIVE_AGENT_OIDC_TOKEN_ENDPOINT"] {
            assert!(
                FORWARDED_VARS.contains(&var),
                "{var} must be forwarded into every agent's env"
            );
        }
        let mut out = String::new();
        push_forwarded_var_options(
            &mut out,
            &[
                ("HIVE_AGENT_NATS_URL", "nats://10.42.0.1:4222".to_string()),
                (
                    "HIVE_AGENT_OIDC_TOKEN_ENDPOINT",
                    "https://auth.t.local/api/oidc/token".to_string(),
                ),
            ],
        );
        assert_eq!(
            out,
            "              services.hyperhive.agent.queue.natsUrl = \"nats://10.42.0.1:4222\";\n\
             \x20             services.hyperhive.agent.queue.tokenEndpoint = \"https://auth.t.local/api/oidc/token\";\n"
        );
    }

    /// `swarm-metrics` is installed only when the option is set, and reads
    /// the env var at runtime, so the metrics store's origin needs both halves.
    #[test]
    fn the_metrics_url_is_forwarded_as_both_env_and_option() {
        assert!(FORWARDED_VARS.contains(&"HIVE_AGENT_METRICS_URL"));
        let mut out = String::new();
        push_forwarded_var_options(
            &mut out,
            &[(
                "HIVE_AGENT_METRICS_URL",
                "https://metrics.t.local".to_string(),
            )],
        );
        assert_eq!(
            out,
            "              services.hyperhive.agent.metrics.url = \"https://metrics.t.local\";\n"
        );
    }

    #[test]
    fn require_service_urls_accepts_a_rendered_forge_url() {
        require_service_urls(&[
            ("HIVE_FORGE_URL", "http://forge.example.test".to_string()),
            ("HYPERHIVE_SWARM_NAME", "constellat1on".to_string()),
        ])
        .expect("a forwarded forge URL satisfies the requirement");
    }

    #[test]
    fn require_service_urls_refuses_to_write_without_a_forge() {
        // The agent option is nullable, so nothing downstream would complain:
        // every agent would simply come up with no forge login and no way to
        // tell that was unintended. The hive asserts its own requirement
        // because it is the only layer that knows a forge is mandatory.
        let err = require_service_urls(&[("HYPERHIVE_SWARM_NAME", "constellat1on".to_string())])
            .expect_err("a missing forge URL must stop the write");
        assert!(
            err.to_string().contains("HIVE_FORGE_URL is unset"),
            "the error must name the missing variable: {err}"
        );
    }

    #[test]
    fn service_url_options_emit_nothing_when_absent() {
        // No guess when the host doesn't say — the agent option stays `null`
        // ("not configured") and the units that would use it aren't generated,
        // so absence is an absent integration rather than a misdirected one.
        // Required services don't reach here: `require_service_urls` rejects
        // them first.
        // `HIVE_FORGE_PUBLIC_URL` is forwarded as env but carries no option —
        // deliberately picked over a mapped var, since the whole point is a
        // forwarded var the option map does not know about.
        let mut out = String::new();
        push_forwarded_var_options(
            &mut out,
            &[(
                "HIVE_FORGE_PUBLIC_URL",
                "https://forge.example.test".to_string(),
            )],
        );
        assert!(
            out.is_empty(),
            "no option should be emitted for an unmapped forwarded var, got:\n{out}"
        );
    }

    #[test]
    fn service_url_options_escape_quotes() {
        // The value lands inside a nix string literal; an unescaped quote
        // would end the string and change the surrounding config rather than
        // merely corrupting one value.
        let mut out = String::new();
        push_forwarded_var_options(
            &mut out,
            &[("HIVE_FORGE_URL", "http://x/\"; evil = \"".to_string())],
        );
        assert!(
            out.contains("\\\"") && !out.contains("/\";"),
            "quotes in the value must be escaped:\n{out}"
        );
    }

    #[test]
    #[ignore = "mutates process env; races the other render_flake env tests under the parallel runner"]
    fn render_flake_sets_service_url_options_from_forwarded_env() {
        // The forwarded env vars must ALSO become option assignments, because
        // the two are consumed at different times: the option is baked into
        // scripts at build time (forge-avatar-sync's FORGE_URL), the env var is read at
        // runtime. Emitting only the env var leaves the option on its default,
        // which is how a hive ends up with two disagreeing answers for the same
        // URL.
        //
        // Asserts PLACEMENT, not just presence: an option line rendered outside
        // the per-agent module block would appear in the file and change
        // nothing. It has to land before the environment blocks that close the
        // module out.
        //
        // Serialised against every other env-mutating test in the crate:
        // `render_flake_forwards_env_into_global_environment` sets HIVE_FORGE_URL
        // too, and both land in the one test binary.
        let _env = crate::test_env::lock();
        // SAFETY: serialised by the guard above; restored before returning.
        unsafe {
            std::env::set_var("HIVE_FORGE_URL", "http://forge.example.test");
            std::env::set_var("HIVE_MATRIX_URL", "http://matrix.example.test");
        }
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        unsafe {
            std::env::remove_var("HIVE_FORGE_URL");
            std::env::remove_var("HIVE_MATRIX_URL");
        }
        let forge_opt_at = out
            .find("services.hyperhive.agent.forge.url = \"http://forge.example.test\"")
            .expect("services.hyperhive.agent.forge.url must be rendered from HIVE_FORGE_URL");
        let matrix_opt_at = out
            .find("services.hyperhive.agent.matrix.url = \"http://matrix.example.test\"")
            .expect("services.hyperhive.agent.matrix.url must be rendered from HIVE_MATRIX_URL");
        let env_block_at = out
            .find("environment.variables = {")
            .expect("per-agent environment.variables block must exist");
        assert!(
            forge_opt_at < env_block_at && matrix_opt_at < env_block_at,
            "service URL options must land inside the per-agent module block:\n{out}"
        );
    }

    #[test]
    fn render_flake_agent_input_points_at_local_applied_mirror() {
        // The agent config input references the LOCAL applied mirror
        // (`git+file://<applied>`), NOT the forge — so the meta flake
        // resolves entirely from on-disk state and boot never depends on the
        // forge being reachable. The forge `agent-configs/<n>` repos stay the
        // review surface, but they are not the flake's build input.
        let out = render_flake(
            "github:example/hyperhive",
            "path:/nix/store/bbbb-hyperhive-docs-source",
            "path:/nix/store/aaaa-nixpkgs-source",
            None,
            8000,
            &[sample_spec("alice", 9001)],
        );
        let want = format!(
            "agent-alice.url = \"git+file://{}\"",
            crate::paths::applied_dir("alice").display()
        );
        assert!(
            out.contains(&want),
            "expected the agent input to point at the local applied mirror ({want}):\n{out}"
        );
        assert!(
            !out.contains("git+http"),
            "no forge git+http URL must remain in the rendered flake:\n{out}"
        );
    }

    #[test]
    fn render_flake_embeds_hive_ca_when_signalled() {
        // When the hyperhive TLS module signals a trust bundle via
        // HIVE_TLS_CA_PATH (and the cert exists), the agent module must
        // trust it at build time via security.pki.certificateFiles. Absent
        // the signal, no reference is emitted (so the flake doesn't point at
        // a file that was never embedded).
        //
        // Serialised against every other env-mutating test in the crate. Taken
        // before the fixture files so the guard covers the whole window in
        // which HIVE_TLS_CA_PATH / HIVE_PEER_CA_PATHS are perturbed.
        let _env = crate::test_env::lock();
        let ca_file = std::env::temp_dir().join(format!("hive-ca-test-{}.pem", std::process::id()));
        std::fs::write(
            &ca_file,
            "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n",
        )
        .expect("write temp CA");

        let render = || {
            render_flake(
                "github:example/hyperhive",
                "path:/nix/store/bbbb-hyperhive-docs-source",
                "path:/nix/store/aaaa-nixpkgs-source",
                None,
                8000,
                &[sample_spec("alice", 9001)],
            )
        };

        // A leftover peer-CA file + the env var that names it. Both must be
        // inert: the store's root rides in the hive's own trust bundle, so
        // nothing per-peer is embedded.
        let stale_peer =
            std::env::temp_dir().join(format!("peer-ca0-test-{}.pem", std::process::id()));
        std::fs::write(
            &stale_peer,
            "-----BEGIN CERTIFICATE-----\np0\n-----END CERTIFICATE-----\n",
        )
        .expect("write stale peer CA");

        // SAFETY: serialised by the guard at the top of this test; restored
        // before returning.
        unsafe {
            std::env::remove_var("HIVE_PEER_CA_PATHS");
            std::env::set_var("HIVE_TLS_CA_PATH", &ca_file);
        }
        let with_ca = render();
        unsafe {
            std::env::set_var("HIVE_PEER_CA_PATHS", stale_peer.display().to_string());
        }
        let with_stale_peer_env = render();
        // The stale var alone, with no trust bundle: must produce nothing.
        unsafe {
            std::env::remove_var("HIVE_TLS_CA_PATH");
        }
        let stale_peer_env_only = render();
        unsafe {
            std::env::remove_var("HIVE_PEER_CA_PATHS");
        }
        let without_ca = render();
        let _ = std::fs::remove_file(&ca_file);
        let _ = std::fs::remove_file(&stale_peer);

        assert!(
            with_ca.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
            "CA cert must be wired into certificateFiles when signalled:\n{with_ca}"
        );
        // The regression guard, scoped to the CA list rather than the whole
        // render. Comparing the two flakes wholesale looks stronger and is
        // actually FLAKY: `cargo test` runs these in parallel threads and
        // sibling tests mutate process env (OTEL, forge URLs) between the
        // two `render()` calls, so a whole-output equality assertion fails
        // on changes that have nothing to do with this test.
        assert!(
            with_stale_peer_env.contains("security.pki.certificateFiles = [ ./hive-ca.pem ]"),
            "HIVE_PEER_CA_PATHS must not change the CA list — the store's \
             root inside the hive's own trust bundle is the only anchor \
             embedded:\n{with_stale_peer_env}"
        );
        assert!(
            !with_stale_peer_env.contains("peer-ca-"),
            "no peer-ca-<N>.pem may be embedded, however HIVE_PEER_CA_PATHS \
             is set:\n{with_stale_peer_env}"
        );
        assert!(
            !stale_peer_env_only.contains("security.pki.certificateFiles"),
            "a stale HIVE_PEER_CA_PATHS must not resurrect a certificateFiles \
             reference on its own:\n{stale_peer_env_only}"
        );
        assert!(
            !without_ca.contains("security.pki.certificateFiles"),
            "no certificateFiles reference without any CA signal:\n{without_ca}"
        );
    }

    #[test]
    fn render_flake_injects_otel_when_signalled() {
        // services.hyperhive.otel.* -> HYPERHIVE_OTEL_* on hive-c0re's unit
        // -> injected as build-time services.hyperhive.agent.otel.* into every agent. With
        // no endpoint signal, no services.hyperhive.agent.otel lines are emitted (agents
        // keep the the harness modules disabled default).
        //
        // Serialised against every other env-mutating test in the crate.
        // `stats::otel_metrics::tests` perturbs HYPERHIVE_OTEL_ENDPOINT too,
        // and lands in this same test binary — "no other test asserts on
        // these" was true of this module and false of the process.
        let _env = crate::test_env::lock();
        // SAFETY: serialised by the guard above; restored before returning.
        let render = || {
            render_flake(
                "github:example/hyperhive",
                "path:/nix/store/bbbb-hyperhive-docs-source",
                "path:/nix/store/aaaa-nixpkgs-source",
                None,
                8000,
                &[sample_spec("alice", 9001)],
            )
        };
        unsafe {
            std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
            std::env::set_var("HYPERHIVE_OTEL_ENDPOINT", "https://c.example/otel");
            std::env::set_var("HYPERHIVE_OTEL_PROTOCOL", "grpc");
        }
        let on_minimal = render();
        unsafe {
            std::env::set_var(
                "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES",
                "deployment.environment=prod",
            );
        }
        let on_full = render();
        unsafe {
            std::env::remove_var("HYPERHIVE_OTEL_ENDPOINT");
            std::env::remove_var("HYPERHIVE_OTEL_PROTOCOL");
            std::env::remove_var("HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES");
        }
        let off = render();

        assert!(
            on_minimal.contains("services.hyperhive.agent.otel.enable = true;"),
            "otel enable must be injected:\n{on_minimal}"
        );
        assert!(
            on_minimal
                .contains("services.hyperhive.agent.otel.endpoint = \"https://c.example/otel\";"),
            "otel endpoint must be injected:\n{on_minimal}"
        );
        assert!(
            on_minimal.contains("services.hyperhive.agent.otel.protocol = \"grpc\";"),
            "otel protocol must be injected:\n{on_minimal}"
        );
        // Optional fields absent when unset.
        assert!(
            !on_minimal.contains("services.hyperhive.agent.otel.extraResourceAttributes"),
            "extraResourceAttributes must not appear when unset:\n{on_minimal}"
        );

        assert!(
            on_full.contains(
                "services.hyperhive.agent.otel.extraResourceAttributes = \"deployment.environment=prod\";"
            ),
            "extraResourceAttributes must be injected when set:\n{on_full}"
        );

        assert!(
            !off.contains("services.hyperhive.agent.otel"),
            "no otel lines when disabled:\n{off}"
        );
    }

    #[test]
    fn render_flake_injects_github_disable_only_when_signalled() {
        // services.hyperhive.swarm.github.enable = false -> HYPERHIVE_GITHUB_DISABLED
        // on hive-c0re's unit -> `services.hyperhive.agent.github.enable = false` injected into
        // every agent. On by default, so nothing is emitted unless disabled.
        //
        // Serialised against every other env-mutating test in the crate — the
        // variable is this module's alone, but the process it lives in is not.
        let _env = crate::test_env::lock();
        // SAFETY: serialised by the guard above; restored before returning.
        let render = || {
            render_flake(
                "github:example/hyperhive",
                "path:/nix/store/bbbb-hyperhive-docs-source",
                "path:/nix/store/aaaa-nixpkgs-source",
                None,
                8000,
                &[sample_spec("alice", 9001)],
            )
        };
        unsafe {
            std::env::remove_var("HYPERHIVE_GITHUB_DISABLED");
        }
        let on_default = render();
        unsafe {
            std::env::set_var("HYPERHIVE_GITHUB_DISABLED", "1");
        }
        let disabled = render();
        unsafe {
            std::env::remove_var("HYPERHIVE_GITHUB_DISABLED");
        }
        assert!(
            !on_default.contains("services.hyperhive.agent.github.enable"),
            "github.enable must not be emitted by default (agents keep the true default):\n{on_default}"
        );
        assert!(
            disabled.contains("services.hyperhive.agent.github.enable = false;"),
            "github.enable = false must be injected when the host disables it:\n{disabled}"
        );
    }

    /// The agent reads its memory cap at boot from the copy
    /// `lifecycle::write_dropins` hands it, so the cap is no build input.
    #[test]
    fn render_hive_flake_is_identical_across_memory_caps() {
        let hive = |cap: &str| HiveEnv {
            agent_memory_max: cap.to_owned(),
            ..HiveEnv::default()
        };
        let agents = [sample_spec("alice", 9001)];
        let small = render_hive_flake(&hive("4G"), &agents);
        let large = render_hive_flake(&hive("16G"), &agents);
        assert_eq!(
            small, large,
            "the rendered flake must not depend on the memory cap"
        );
    }
}