hyperhive/hive-c0re/src/lifecycle/tests.rs

//! Unit tests for the lifecycle module (moved verbatim from the old
//! single-file `lifecycle.rs` `#[cfg(test)]` block).

use super::*;

/// Regression test: `setup_proposed` must seed both agent.nix and flake.nix
/// in the initial commit. Before commit 5b5a93e flake.nix was missing from
/// the scaffold, requiring manual creation (seen with the damocles agent).
///
/// Exercises the template arm: there is no forge to clone from in a test
/// (`forge::is_present` needs the priv socket), so `setup_proposed` falls
/// through to `seed_template` — which is the arm this asserts about.
#[tokio::test]
async fn setup_proposed_seeds_flake_nix() {
    let dir = tempfile::tempdir().expect("tempdir");
    let proposed = dir.path().join("proposed");
    setup_proposed(&proposed, "test-agent")
        .await
        .expect("setup_proposed");

    // Both files must exist on disk.
    assert!(proposed.join("agent.nix").exists(), "agent.nix missing");
    assert!(proposed.join("flake.nix").exists(), "flake.nix missing");

    // flake.nix must export nixosModules.default (the meta-flake contract).
    let flake = std::fs::read_to_string(proposed.join("flake.nix")).unwrap();
    assert!(
        flake.contains("nixosModules.default"),
        "flake.nix does not export nixosModules.default"
    );

    // Both files must be tracked in the initial git commit.
    let out = git_command()
        .current_dir(&proposed)
        .args(["show", "--name-only", "--format=", "HEAD"])
        .output()
        .await
        .expect("git show");
    let tracked = String::from_utf8_lossy(&out.stdout);
    assert!(tracked.contains("agent.nix"), "agent.nix not committed");
    assert!(tracked.contains("flake.nix"), "flake.nix not committed");
}

#[test]
fn bridge_gateway_ip_extracts_verbatim_address() {
    // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
    // canonical network — the gateway is the address before the `/`.
    assert_eq!(
        bridge_gateway_ip("10.42.0.1/24").as_deref(),
        Some("10.42.0.1")
    );
    // Non-`.1` operator override: the gateway is wherever the bridge is.
    assert_eq!(
        bridge_gateway_ip("10.42.0.254/24").as_deref(),
        Some("10.42.0.254")
    );
    assert_eq!(
        bridge_gateway_ip("172.30.0.1/16").as_deref(),
        Some("172.30.0.1")
    );
}

#[test]
fn bridge_gateway_ip_rejects_bad_input() {
    assert!(bridge_gateway_ip("notanip/24").is_none());
    assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix
    assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32
    assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255
    assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
}

/// The presence control for the two rejection tests below: with both
/// variables set and well-formed, the settings are built. Without this,
/// a `network_isolation_from_vars` that rejected *everything* would pass
/// every absence assertion and look like a working guard.
#[test]
fn network_isolation_accepts_a_well_formed_pair() {
    let iso = network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/24"))
        .expect("well-formed bridge + subnet must be accepted");
    assert_eq!(iso.bridge, "hive0");
    // The gateway is the verbatim bridge address, prefix stripped.
    assert_eq!(iso.gateway_ip, "10.42.0.1");
}

/// A missing or empty variable is fatal, not a fallback to the host
/// netns. Empty is tested alongside unset because `std::env::var` on a
/// variable set to `""` returns `Ok("")`, so treating only `None` as
/// missing would let an empty value through.
#[test]
fn network_isolation_rejects_missing_or_empty_vars() {
    assert!(network_isolation_from_vars(None, Some("10.42.0.1/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), None).is_err());
    assert!(network_isolation_from_vars(None, None).is_err());
    assert!(network_isolation_from_vars(Some(""), Some("10.42.0.1/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("")).is_err());
}

/// A malformed subnet is fatal too. Previously this logged a warning and
/// silently produced a container on the host netns — a dropped security
/// boundary with nothing in the journal saying so.
#[test]
fn network_isolation_rejects_a_malformed_subnet() {
    assert!(network_isolation_from_vars(Some("hive0"), Some("notanip/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/33")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.999/24")).is_err());
}

/// The error has to name the variable an operator must fix — these are
/// read at startup, so the message is the whole diagnostic.
#[test]
fn network_isolation_errors_name_the_offending_variable() {
    let e = network_isolation_from_vars(None, Some("10.42.0.1/24")).unwrap_err();
    assert!(
        format!("{e:#}").contains("HIVE_NETWORK_BRIDGE"),
        "bridge error must name the variable, got: {e:#}"
    );
    let e = network_isolation_from_vars(Some("hive0"), None).unwrap_err();
    assert!(
        format!("{e:#}").contains("HIVE_NETWORK_SUBNET"),
        "subnet error must name the variable, got: {e:#}"
    );
    let e = network_isolation_from_vars(Some("hive0"), Some("nope/24")).unwrap_err();
    let msg = format!("{e:#}");
    assert!(
        msg.contains("HIVE_NETWORK_SUBNET") && msg.contains("nope/24"),
        "malformed-subnet error must name the variable and the bad value, got: {msg}"
    );
}

/// `setup_proposed` is idempotent: calling it on an existing repo is a
/// no-op (the fresh guard skips all writes).
#[tokio::test]
async fn setup_proposed_idempotent() {
    let dir = tempfile::tempdir().expect("tempdir");
    let proposed = dir.path().join("proposed");
    setup_proposed(&proposed, "test-agent")
        .await
        .expect("first call");
    // Second call must not error even though .git already exists.
    setup_proposed(&proposed, "test-agent")
        .await
        .expect("second call");
    // Still one commit.
    let out = git_command()
        .current_dir(&proposed)
        .args(["rev-list", "--count", "HEAD"])
        .output()
        .await
        .expect("git rev-list");
    let count = String::from_utf8_lossy(&out.stdout).trim().to_owned();
    assert_eq!(
        count, "1",
        "expected exactly one commit after idempotent call"
    );
}

/// Build a two-commit repo and return `(dir, repo_path, first, second)`.
async fn two_commit_repo() -> (tempfile::TempDir, std::path::PathBuf, String, String) {
    let dir = tempfile::tempdir().expect("tempdir");
    let repo = dir.path().join("proposed");
    setup_proposed(&repo, "test-agent")
        .await
        .expect("setup_proposed");
    let first = git_rev_parse(&repo, "HEAD").await.expect("rev-parse first");

    std::fs::write(repo.join("second.txt"), "second").expect("write second.txt");
    git(&repo, &["add", "second.txt"]).await.expect("git add");
    git(
        &repo,
        &[
            "-c",
            "user.name=test",
            "-c",
            "user.email=test@example.com",
            "commit",
            "-m",
            "second",
        ],
    )
    .await
    .expect("git commit");
    let second = git_rev_parse(&repo, "HEAD")
        .await
        .expect("rev-parse second");

    assert_ne!(first, second, "second commit did not advance HEAD");
    (dir, repo, first, second)
}

/// `git_is_ancestor` answers reachability in both directions. This is the
/// check that decides whether moving a branch is a real fast-forward or a
/// rewind that discards commits.
#[tokio::test]
async fn is_ancestor_distinguishes_direction() {
    let (_dir, repo, first, second) = two_commit_repo().await;

    assert!(
        git_is_ancestor(&repo, &first, &second)
            .await
            .expect("ancestor check forward"),
        "first must be an ancestor of second"
    );
    assert!(
        !git_is_ancestor(&repo, &second, &first)
            .await
            .expect("ancestor check reverse"),
        "second must NOT be an ancestor of first"
    );
}

/// Regression test for the deploy path that ate a committed agent config:
/// `git_update_ref_cas` must refuse to move a ref whose current value is not
/// the expected one, and must leave the ref untouched when it refuses. A bare
/// `update-ref` accepts that move and silently drops everything in between.
#[tokio::test]
async fn update_ref_cas_refuses_stale_expectation() {
    let (_dir, repo, first, second) = two_commit_repo().await;

    // Park a ref at `first`, then CAS it forward with the correct old value.
    git_update_ref(&repo, "refs/heads/cas", &first)
        .await
        .expect("plant ref");
    git_update_ref_cas(&repo, "refs/heads/cas", &second, &first)
        .await
        .expect("CAS with the correct old value must succeed");
    assert_eq!(
        git_rev_parse(&repo, "refs/heads/cas").await.unwrap(),
        second
    );

    // Now retry with the SAME (now stale) expectation, as a racing deploy
    // holding a pre-merge sha would: it must fail rather than rewind.
    assert!(
        git_update_ref_cas(&repo, "refs/heads/cas", &first, &first)
            .await
            .is_err(),
        "CAS accepted a stale old value"
    );
    assert_eq!(
        git_rev_parse(&repo, "refs/heads/cas").await.unwrap(),
        second,
        "ref moved even though the CAS failed"
    );
}

/// `systemctl is-active` prints the state with a trailing newline, which is
/// the whole reason this parser trims rather than comparing raw.
#[test]
fn unit_state_reads_what_is_active_prints() {
    assert_eq!(UnitState::from_is_active("active\n"), UnitState::Active);
    assert_eq!(UnitState::from_is_active("failed\n"), UnitState::Failed);
    assert_eq!(UnitState::from_is_active("inactive\n"), UnitState::Other);
    // No trailing newline, in case the capture ever changes shape.
    assert_eq!(UnitState::from_is_active("failed"), UnitState::Failed);
}

/// The asymmetry that matters: an unrecognised state must fall to `Other`,
/// never to `Failed`.
///
/// `Failed` is what the dashboard will render as *this agent gave up*, so a
/// wrong `Failed` invents an incident, while a wrong `Other` merely fails to
/// distinguish one from a deliberate stop — the behaviour we have today.
/// Guessing in the safe direction is the property, not an implementation
/// detail of the `match`.
#[test]
fn an_unknown_state_is_never_reported_as_failed() {
    for s in [
        "activating",
        "deactivating",
        "reloading",
        "maintenance",
        "unknown",
        "",
        "Failed",  // capitalised: systemd prints lowercase, so this is not a state
        "failed*", // `is-active` never emits this; a substring match would take it
    ] {
        assert_eq!(
            UnitState::from_is_active(s),
            UnitState::Other,
            "{s:?} must not be read as a give-up"
        );
    }
}

/// A failed `nixos-container destroy` whose container is still listed fails,
/// and keeps hive-priv's error in the chain. The destroy DAG gates the state
/// purge on this call, so this is the case that must not read as success.
#[test]
fn failed_destroy_with_the_container_still_listed_fails() {
    let err = confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed (exit status: 1)"),
        "h-doomed",
        Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
    )
    .expect_err("a surviving container must fail the destroy");
    let chain = format!("{err:#}");
    assert!(chain.contains("h-doomed still exists"), "{chain}");
    assert!(chain.contains("exit status: 1"), "{chain}");
}

/// A container missing from the list is gone, whatever the destroy exit code
/// said, so the destroy succeeds.
#[test]
fn failed_destroy_with_the_container_gone_succeeds() {
    confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed"),
        "h-doomed",
        Ok(vec!["h-other".to_owned()]),
    )
    .expect("an absent container is a completed destroy");
    // Control: the same list with the container in it fails, so the pass
    // above comes from the absence, not from the helper accepting anything.
    assert!(
        confirm_gone_after_failed_destroy(
            anyhow::anyhow!("nixos-container destroy h-doomed failed"),
            "h-doomed",
            Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
        )
        .is_err()
    );
}

/// An unreadable list proves nothing about the container, so the destroy
/// fails rather than guessing it is gone.
#[test]
fn failed_destroy_with_an_unreadable_list_fails() {
    let err = confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed"),
        "h-doomed",
        Err(anyhow::anyhow!("connect to hive-priv socket")),
    )
    .expect_err("an unreadable list must not pass for an absent container");
    assert!(format!("{err:#}").contains("cannot confirm"), "{err:#}");
}

#[test]
fn agent_names_strips_the_container_prefix() {
    let names = agent_names(Ok(vec![format!("{AGENT_PREFIX}iris"), "other".to_owned()]))
        .expect("a readable list");
    assert_eq!(names, vec!["iris"]);
}

/// A failed list is not an empty hive: the error reaches the caller.
#[test]
fn agent_names_propagates_an_unreadable_list() {
    let err = agent_names(Err(anyhow::anyhow!("connect to hive-priv socket")))
        .expect_err("an unreadable list must not read as zero agents");
    assert!(format!("{err:#}").contains("hive-priv"), "{err:#}");
}

fn is_dead(pid: &str) -> bool {
    // A killed process that is not our child stays a zombie until its own
    // parent reaps it, so a zombie counts as dead.
    match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
        Err(_) => true,
        Ok(stat) => stat
            .rsplit_once(") ")
            .is_some_and(|(_, rest)| rest.starts_with('Z')),
    }
}

/// A child still running at the limit comes back `None`, and the kill reaches
/// its whole process group: the backgrounded grandchild dies too, not just the
/// shell that started it.
#[tokio::test]
async fn a_prebuild_past_its_limit_is_killed_with_its_group() {
    use std::time::{Duration, Instant};
    let dir = tempfile::tempdir().expect("tempdir");
    let pidfile = dir.path().join("grandchild");
    let mut child = Command::new("sh")
        .arg("-c")
        .arg(format!("sleep 600 & echo $! > {}; wait", pidfile.display()))
        .process_group(0)
        .spawn()
        .expect("spawn sh");

    let started = Instant::now();
    let status = wait_bounded(&mut child, Duration::from_secs(2))
        .await
        .expect("wait_bounded");
    assert!(status.is_none(), "must time out, got {status:?}");
    assert!(
        started.elapsed() < Duration::from_mins(1),
        "the limit did not bound the wait"
    );

    let pid = std::fs::read_to_string(&pidfile).expect("grandchild pid");
    let pid = pid.trim();
    let deadline = Instant::now() + Duration::from_secs(5);
    while !is_dead(pid) && Instant::now() < deadline {
        tokio::time::sleep(Duration::from_millis(50)).await;
    }
    assert!(is_dead(pid), "grandchild {pid} outlived the timeout");
}

/// The control: a child that exits inside its limit comes back with its
/// status.
#[tokio::test]
async fn a_prebuild_inside_its_limit_returns_its_status() {
    let mut child = Command::new("sh")
        .arg("-c")
        .arg("exit 3")
        .process_group(0)
        .spawn()
        .expect("spawn sh");
    let status = wait_bounded(&mut child, std::time::Duration::from_mins(1))
        .await
        .expect("wait_bounded")
        .expect("a fast child must not time out");
    assert_eq!(status.code(), Some(3));
}

/// The setup path creates the agent's `work/` dir beside `state/`, so the
/// bind source exists before nspawn starts the container.
#[test]
fn ensure_state_dir_creates_the_work_dir() {
    let root = tempfile::tempdir().expect("tempdir");
    let notes_dir = root.path().join("state");
    ensure_state_dir(&notes_dir).expect("ensure_state_dir");
    assert!(notes_dir.is_dir());
    assert!(root.path().join(crate::paths::WORK_SUBDIR).is_dir());
}

/// Real build failures (`error:`) and both of nix's own warning spellings
/// must stay WARN; routine progress chatter is the only thing demoted.
/// Regression guard for a first-pass predicate that only matched
/// `warning:` and silently buried `error:` lines at INFO.
#[test]
fn is_actionable_nix_line_keeps_warnings_and_errors_demotes_chatter() {
    let actionable = [
        "warning: Lock file entry is unlocked",
        "  warning: leading whitespace",
        "evaluation warning: this will be an error in a future release",
        "error: builder for '/nix/store/foo.drv' failed with exit code 1",
        "error: 1 dependencies of derivation '/nix/store/bar.drv' failed to build",
    ];
    for line in actionable {
        assert!(is_actionable_nix_line(line), "{line}");
    }

    let chatter = [
        "copying 3 paths",
        "these 2 derivations will be built:",
        "fetching git input 'git+file:///var/lib/hyperhive'",
        "structuredAttrs is enabled",
    ];
    for line in chatter {
        assert!(!is_actionable_nix_line(line), "{line}");
    }
}

#[tokio::test]
async fn a_hung_systemctl_reads_as_unknown_within_the_bound() {
    let started = tokio::time::Instant::now();
    let got = output_within(
        std::time::Duration::from_millis(20),
        std::future::pending::<std::io::Result<()>>(),
        &["is-active"],
    )
    .await;
    assert!(got.is_none());
    assert!(started.elapsed() < std::time::Duration::from_secs(5));
}

#[tokio::test]
async fn a_systemctl_answer_inside_the_bound_passes_through() {
    let got = output_within(
        std::time::Duration::from_secs(5),
        async { Ok(7_u8) },
        &["is-active"],
    )
    .await;
    assert_eq!(got, Some(7));
}

/// A child that outlives the deadline yields an error naming the command and
/// the deadline, and the child and its grandchild are both killed rather than
/// orphaned.
#[tokio::test]
async fn run_with_deadline_times_out_and_kills_the_process_group() {
    let dir = tempfile::tempdir().expect("tempdir");
    let (shpid, sleeppid) = (dir.path().join("sh.pid"), dir.path().join("sleep.pid"));
    let mut cmd = tokio::process::Command::new("sh");
    cmd.arg("-c").arg(format!(
        "echo $$ > {}; sleep 300 & echo $! > {}; wait",
        shpid.display(),
        sleeppid.display()
    ));
    let err = run_with_deadline(
        &mut cmd,
        std::time::Duration::from_millis(500),
        "fake fetch",
    )
    .await
    .expect_err("a sleeping child must hit the deadline");
    let msg = format!("{err:#}");
    assert!(msg.contains("fake fetch timed out after 0.5s"), "{msg}");

    for file in [&shpid, &sleeppid] {
        // The pid file proves the process existed before the kill.
        let pid = std::fs::read_to_string(file)
            .expect("process wrote its pid before the deadline")
            .trim()
            .to_owned();
        let proc = std::path::PathBuf::from("/proc").join(&pid);
        let mut gone = false;
        for _ in 0..100 {
            if !proc.exists() {
                gone = true;
                break;
            }
            tokio::time::sleep(std::time::Duration::from_millis(50)).await;
        }
        assert!(gone, "pid {pid} ({}) still running", file.display());
    }
}

/// Presence control for the timeout test: a child that finishes in time
/// returns its output.
#[tokio::test]
async fn run_with_deadline_returns_output_of_a_fast_child() {
    let mut cmd = tokio::process::Command::new("echo");
    cmd.arg("hi");
    let out = run_with_deadline(&mut cmd, std::time::Duration::from_secs(30), "echo")
        .await
        .expect("echo finishes");
    assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi");
}

/// The per-host values reach the container as facts: the dashboard port and
/// pinned claude from the hive's config, the OTEL labels and the queue
/// address from the daemon's environment.
#[test]
fn hive_facts_carry_the_per_host_values() {
    let hive = HiveEnv {
        dashboard_port: 7123,
        claude_code_path: Some("/nix/store/0abc-claude-code".to_owned()),
        ..HiveEnv::default()
    };
    let facts = hive_facts_from(&hive, |name| match name {
        "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES" => Some("team=a".to_owned()),
        "HIVE_AGENT_NATS_URL" => Some("tls://nats.t.local:4222".to_owned()),
        _ => None,
    });
    assert_eq!(facts.dashboard_port, Some(7123));
    assert_eq!(
        facts.claude_code_path.as_deref(),
        Some("/nix/store/0abc-claude-code")
    );
    assert_eq!(
        facts.otel_extra_resource_attributes.as_deref(),
        Some("team=a")
    );
    assert_eq!(
        facts.agent_nats_url.as_deref(),
        Some("tls://nats.t.local:4222")
    );
}

/// Unset and empty both leave a fact out, so the agent falls back to its
/// own default rather than reading an empty value.
#[test]
fn hive_facts_leave_unset_values_out() {
    let facts = hive_facts_from(&HiveEnv::default(), |name| {
        (name == "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES").then(String::new)
    });
    assert_eq!(facts.claude_code_path, None);
    assert_eq!(facts.otel_extra_resource_attributes, None);
    assert_eq!(facts.agent_nats_url, None);
    assert_eq!(facts.name, None);
}