hyperhive/hive-c0re/src/lifecycle/tests.rs

//! Unit tests for the lifecycle module (moved verbatim from the old
//! single-file `lifecycle.rs` `#[cfg(test)]` block).

use super::*;

#[test]
fn bridge_gateway_ip_extracts_verbatim_address() {
    // HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
    // canonical network — the gateway is the address before the `/`.
    assert_eq!(
        bridge_gateway_ip("10.42.0.1/24").as_deref(),
        Some("10.42.0.1")
    );
    // Non-`.1` operator override: the gateway is wherever the bridge is.
    assert_eq!(
        bridge_gateway_ip("10.42.0.254/24").as_deref(),
        Some("10.42.0.254")
    );
    assert_eq!(
        bridge_gateway_ip("172.30.0.1/16").as_deref(),
        Some("172.30.0.1")
    );
}

#[test]
fn bridge_gateway_ip_rejects_bad_input() {
    assert!(bridge_gateway_ip("notanip/24").is_none());
    assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix
    assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32
    assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255
    assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
}

/// The presence control for the two rejection tests below: with both
/// variables set and well-formed, the settings are built. Without this,
/// a `network_isolation_from_vars` that rejected *everything* would pass
/// every absence assertion and look like a working guard.
#[test]
fn network_isolation_accepts_a_well_formed_pair() {
    let iso = network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/24"))
        .expect("well-formed bridge + subnet must be accepted");
    assert_eq!(iso.bridge, "hive0");
    // The gateway is the verbatim bridge address, prefix stripped.
    assert_eq!(iso.gateway_ip, "10.42.0.1");
}

/// A missing or empty variable is fatal, not a fallback to the host
/// netns. Empty is tested alongside unset because `std::env::var` on a
/// variable set to `""` returns `Ok("")`, so treating only `None` as
/// missing would let an empty value through.
#[test]
fn network_isolation_rejects_missing_or_empty_vars() {
    assert!(network_isolation_from_vars(None, Some("10.42.0.1/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), None).is_err());
    assert!(network_isolation_from_vars(None, None).is_err());
    assert!(network_isolation_from_vars(Some(""), Some("10.42.0.1/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("")).is_err());
}

/// A malformed subnet is fatal too. Previously this logged a warning and
/// silently produced a container on the host netns — a dropped security
/// boundary with nothing in the journal saying so.
#[test]
fn network_isolation_rejects_a_malformed_subnet() {
    assert!(network_isolation_from_vars(Some("hive0"), Some("notanip/24")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/33")).is_err());
    assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.999/24")).is_err());
}

/// The error has to name the variable an operator must fix — these are
/// read at startup, so the message is the whole diagnostic.
#[test]
fn network_isolation_errors_name_the_offending_variable() {
    let e = network_isolation_from_vars(None, Some("10.42.0.1/24")).unwrap_err();
    assert!(
        format!("{e:#}").contains("HIVE_NETWORK_BRIDGE"),
        "bridge error must name the variable, got: {e:#}"
    );
    let e = network_isolation_from_vars(Some("hive0"), None).unwrap_err();
    assert!(
        format!("{e:#}").contains("HIVE_NETWORK_SUBNET"),
        "subnet error must name the variable, got: {e:#}"
    );
    let e = network_isolation_from_vars(Some("hive0"), Some("nope/24")).unwrap_err();
    let msg = format!("{e:#}");
    assert!(
        msg.contains("HIVE_NETWORK_SUBNET") && msg.contains("nope/24"),
        "malformed-subnet error must name the variable and the bad value, got: {msg}"
    );
}

/// `systemctl is-active` prints the state with a trailing newline, which is
/// the whole reason this parser trims rather than comparing raw.
#[test]
fn unit_state_reads_what_is_active_prints() {
    assert_eq!(UnitState::from_is_active("active\n"), UnitState::Active);
    assert_eq!(UnitState::from_is_active("failed\n"), UnitState::Failed);
    assert_eq!(UnitState::from_is_active("inactive\n"), UnitState::Other);
    // No trailing newline, in case the capture ever changes shape.
    assert_eq!(UnitState::from_is_active("failed"), UnitState::Failed);
}

/// The asymmetry that matters: an unrecognised state must fall to `Other`,
/// never to `Failed`.
///
/// `Failed` is what the dashboard will render as *this agent gave up*, so a
/// wrong `Failed` invents an incident, while a wrong `Other` merely fails to
/// distinguish one from a deliberate stop — the behaviour we have today.
/// Guessing in the safe direction is the property, not an implementation
/// detail of the `match`.
#[test]
fn an_unknown_state_is_never_reported_as_failed() {
    for s in [
        "activating",
        "deactivating",
        "reloading",
        "maintenance",
        "unknown",
        "",
        "Failed",  // capitalised: systemd prints lowercase, so this is not a state
        "failed*", // `is-active` never emits this; a substring match would take it
    ] {
        assert_eq!(
            UnitState::from_is_active(s),
            UnitState::Other,
            "{s:?} must not be read as a give-up"
        );
    }
}

/// A failed `nixos-container destroy` whose container is still listed fails,
/// and keeps hive-priv's error in the chain. The destroy DAG gates the state
/// purge on this call, so this is the case that must not read as success.
#[test]
fn failed_destroy_with_the_container_still_listed_fails() {
    let err = confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed (exit status: 1)"),
        "h-doomed",
        Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
    )
    .expect_err("a surviving container must fail the destroy");
    let chain = format!("{err:#}");
    assert!(chain.contains("h-doomed still exists"), "{chain}");
    assert!(chain.contains("exit status: 1"), "{chain}");
}

/// A container missing from the list is gone, whatever the destroy exit code
/// said, so the destroy succeeds.
#[test]
fn failed_destroy_with_the_container_gone_succeeds() {
    confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed"),
        "h-doomed",
        Ok(vec!["h-other".to_owned()]),
    )
    .expect("an absent container is a completed destroy");
    // Control: the same list with the container in it fails, so the pass
    // above comes from the absence, not from the helper accepting anything.
    assert!(
        confirm_gone_after_failed_destroy(
            anyhow::anyhow!("nixos-container destroy h-doomed failed"),
            "h-doomed",
            Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
        )
        .is_err()
    );
}

/// An unreadable list proves nothing about the container, so the destroy
/// fails rather than guessing it is gone.
#[test]
fn failed_destroy_with_an_unreadable_list_fails() {
    let err = confirm_gone_after_failed_destroy(
        anyhow::anyhow!("nixos-container destroy h-doomed failed"),
        "h-doomed",
        Err(anyhow::anyhow!("connect to hive-priv socket")),
    )
    .expect_err("an unreadable list must not pass for an absent container");
    assert!(format!("{err:#}").contains("cannot confirm"), "{err:#}");
}

/// The setup path creates the agent's `work/` dir beside `state/`, so the
/// bind source exists before nspawn starts the container.
#[test]
fn ensure_state_dir_creates_the_work_dir() {
    let root = tempfile::tempdir().expect("tempdir");
    let notes_dir = root.path().join("state");
    ensure_state_dir(&notes_dir).expect("ensure_state_dir");
    assert!(notes_dir.is_dir());
    assert!(root.path().join(crate::paths::WORK_SUBDIR).is_dir());
}

/// Real build failures (`error:`) and both of nix's own warning spellings
/// must stay WARN; routine progress chatter is the only thing demoted.
/// Regression guard for a first-pass predicate that only matched
/// `warning:` and silently buried `error:` lines at INFO.
#[test]
fn is_actionable_nix_line_keeps_warnings_and_errors_demotes_chatter() {
    let actionable = [
        "warning: Lock file entry is unlocked",
        "  warning: leading whitespace",
        "evaluation warning: this will be an error in a future release",
        "error: builder for '/nix/store/foo.drv' failed with exit code 1",
        "error: 1 dependencies of derivation '/nix/store/bar.drv' failed to build",
    ];
    for line in actionable {
        assert!(is_actionable_nix_line(line), "{line}");
    }

    let chatter = [
        "copying 3 paths",
        "these 2 derivations will be built:",
        "fetching git input 'git+file:///var/lib/hyperhive'",
        "structuredAttrs is enabled",
    ];
    for line in chatter {
        assert!(!is_actionable_nix_line(line), "{line}");
    }
}

#[tokio::test]
async fn a_hung_systemctl_reads_as_unknown_within_the_bound() {
    let started = tokio::time::Instant::now();
    let got = output_within(
        std::time::Duration::from_millis(20),
        std::future::pending::<std::io::Result<()>>(),
        &["is-active"],
    )
    .await;
    assert!(got.is_none());
    assert!(started.elapsed() < std::time::Duration::from_secs(5));
}

#[tokio::test]
async fn a_systemctl_answer_inside_the_bound_passes_through() {
    let got = output_within(
        std::time::Duration::from_secs(5),
        async { Ok(7_u8) },
        &["is-active"],
    )
    .await;
    assert_eq!(got, Some(7));
}

/// The per-host values reach the container as facts: the dashboard port and
/// pinned claude from the hive's config, the OTEL labels and the queue
/// address from the daemon's environment.
#[test]
fn hive_facts_carry_the_per_host_values() {
    let hive = HiveEnv {
        dashboard_port: 7123,
        claude_code_path: Some("/nix/store/0abc-claude-code".to_owned()),
        ..HiveEnv::default()
    };
    let facts = hive_facts_from(&hive, |name| match name {
        "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES" => Some("team=a".to_owned()),
        "HIVE_AGENT_NATS_URL" => Some("tls://nats.t.local:4222".to_owned()),
        "HIVE_SQUID_PROXY" => Some("10.42.0.1:3128".to_owned()),
        "HIVE_EGRESS_NO_PROXY" => Some("10.42.0.1,t.local".to_owned()),
        _ => None,
    });
    assert_eq!(facts.dashboard_port, Some(7123));
    assert_eq!(
        facts.claude_code_path.as_deref(),
        Some("/nix/store/0abc-claude-code")
    );
    assert_eq!(
        facts.otel_extra_resource_attributes.as_deref(),
        Some("team=a")
    );
    assert_eq!(
        facts.agent_nats_url.as_deref(),
        Some("tls://nats.t.local:4222")
    );
    assert_eq!(facts.squid_proxy.as_deref(), Some("10.42.0.1:3128"));
    assert_eq!(facts.egress_no_proxy.as_deref(), Some("10.42.0.1,t.local"));
}

/// Unset and empty both leave a fact out, so the agent falls back to its
/// own default rather than reading an empty value.
#[test]
fn hive_facts_leave_unset_values_out() {
    let facts = hive_facts_from(&HiveEnv::default(), |name| {
        (name == "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES").then(String::new)
    });
    assert_eq!(facts.claude_code_path, None);
    assert_eq!(facts.otel_extra_resource_attributes, None);
    assert_eq!(facts.agent_nats_url, None);
    assert_eq!(facts.squid_proxy, None);
    assert_eq!(facts.egress_no_proxy, None);
    assert_eq!(facts.name, None);
}

/// A failed `start` leaves its error in the op's build-log row; without it the
/// row is empty. A missing row id records nothing and does not panic.
#[test]
fn a_failed_start_puts_its_error_in_the_op_output() {
    let dir = tempfile::tempdir().expect("tempdir");
    let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
    let id = logs
        .start("updo", "start", "nixos-container start h-updo", None)
        .expect("start");
    let err = anyhow::anyhow!("nixos-container start h-updo failed (exit status: 1): Job failed");
    let journal = Ok("Creating mount point failed: Read-only file system".to_owned());
    record_failure(Some(&logs), Some(id), &err, journal);
    let full = logs.get_full(id).expect("get").expect("row");
    assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
    assert!(
        full.stderr.contains("Read-only file system"),
        "{}",
        full.stderr
    );
    record_failure(Some(&logs), None, &err, Ok(String::new()));
    record_failure(None, Some(id), &err, Ok(String::new()));
}

/// An older hive-priv rejects the op; the start's own error is still recorded.
#[test]
fn an_unknown_journal_op_keeps_the_start_error() {
    let dir = tempfile::tempdir().expect("tempdir");
    let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
    let id = logs
        .start("updo", "start", "nixos-container start h-updo", None)
        .expect("start");
    let err = anyhow::anyhow!("Job failed");
    let unknown = Err(anyhow::anyhow!(
        "parse request: unknown variant `start_failure_log`"
    ));
    record_failure(Some(&logs), Some(id), &err, unknown);
    let full = logs.get_full(id).expect("get").expect("row");
    assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
    assert!(!full.stderr.contains("unknown variant"), "{}", full.stderr);
}

/// One logged event: its level and each field as `name=value`.
type Event = (tracing::Level, Vec<(String, String)>);

/// Every event a block logged.
#[derive(Clone, Default)]
struct Events(std::sync::Arc<std::sync::Mutex<Vec<Event>>>);

impl tracing::Subscriber for Events {
    fn enabled(&self, _: &tracing::Metadata<'_>) -> bool {
        true
    }
    fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id {
        tracing::span::Id::from_u64(1)
    }
    fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {}
    fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {}
    fn event(&self, event: &tracing::Event<'_>) {
        let mut fields = Vec::new();
        event.record(
            &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
                fields.push((field.name().to_owned(), format!("{value:?}")));
            },
        );
        self.0
            .lock()
            .unwrap()
            .push((*event.metadata().level(), fields));
    }
    fn enter(&self, _: &tracing::span::Id) {}
    fn exit(&self, _: &tracing::span::Id) {}
}

fn field<'a>(fields: &'a [(String, String)], name: &str) -> Option<&'a str> {
    fields
        .iter()
        .find(|(n, _)| n == name)
        .map(|(_, v)| v.as_str())
}

/// A build's log lines say which build they belong to, so the log store can
/// group them: a `start` line, the output lines, then a `finish` line with the
/// status and runtime. A nix error is a warning, chatter stays info, and a
/// failed finish is a warning.
#[test]
fn a_build_logs_its_start_its_lines_and_its_finish_under_one_build_id() {
    use hive_priv_sock::PrivStream;
    let events = Events::default();
    let _guard = tracing::subscriber::set_default(events.clone());
    let tag = BuildTag::new(Some(7), "alice", "update", "nixos-container update h-alice");
    tag.started();
    tag.line(PrivStream::Stdout, "copying 3 paths");
    tag.line(PrivStream::Stderr, "error: builder failed");
    tag.finished(false, std::time::Duration::from_secs(12));

    let logged = events.0.lock().unwrap();
    assert_eq!(logged.len(), 4);
    for (_, fields) in logged.iter() {
        assert_eq!(field(fields, "build_id"), Some("7"));
        assert_eq!(field(fields, "agent"), Some("alice"));
        assert_eq!(field(fields, "kind"), Some("update"));
    }
    let levels: Vec<_> = logged.iter().map(|(l, _)| *l).collect();
    assert_eq!(
        levels,
        [
            tracing::Level::INFO,
            tracing::Level::INFO,
            tracing::Level::WARN,
            tracing::Level::WARN
        ]
    );
    assert_eq!(field(&logged[0].1, "event"), Some("\"start\""));
    assert_eq!(field(&logged[1].1, "event"), None);
    assert_eq!(field(&logged[3].1, "event"), Some("\"finish\""));
    assert_eq!(field(&logged[3].1, "status"), Some("\"fail\""));
    assert_eq!(field(&logged[3].1, "runtime_secs"), Some("12"));
}

/// Without the build-log store there is no id; the lines still carry the
/// agent and kind, with an empty `build_id`.
#[test]
fn a_build_without_a_log_row_has_an_empty_build_id() {
    let events = Events::default();
    let _guard = tracing::subscriber::set_default(events.clone());
    BuildTag::new(None, "alice", "create", "nixos-container create h-alice").started();
    let logged = events.0.lock().unwrap();
    assert_eq!(field(&logged[0].1, "build_id"), Some(""));
    assert_eq!(field(&logged[0].1, "agent"), Some("alice"));
}

#[test]
fn stop_kind_is_leaked_once_per_agent() {
    let first = stop_kind("stopkind-a");
    assert_eq!(first, "agent_stop_failed_stopkind-a");
    assert!(std::ptr::eq(first, stop_kind("stopkind-a")));
    assert!(!std::ptr::eq(first, stop_kind("stopkind-b")));
}

#[test]
fn stop_kind_survives_a_clean_stop_between_failures() {
    let failed = |guards: &mut std::collections::HashMap<_, _>| {
        let kind = stop_kind("stopkind-c");
        guards.insert(
            "stopkind-c".to_owned(),
            crate::warnings::set_warning(kind, "crit", "stop failed"),
        );
        kind
    };
    let mut guards = std::collections::HashMap::new();
    let first = failed(&mut guards);
    guards.remove("stopkind-c");
    assert!(std::ptr::eq(first, failed(&mut guards)));
}