test the remaining job-queue executors through the host seam

hive-c0re/src/job_queue/exec/tests.rs

@@ -3,17 +3,22 @@
 //! the host calls (order and arguments), the executor's outcome, and the
 //! coordinator state the executor is meant to leave behind.
 
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use hive_jobq::NodeId;
 
 use super::{
-    Host, run_drain, run_pause_drain, run_pause_signal, run_prebuild, run_reconcile,
-    run_set_wanted, run_signal, run_stop, run_stop_for_update, run_swap,
+    Host, run_create, run_destroy_bookkeeping, run_destroy_container, run_drain, run_emit_rebuilt,
+    run_forge_sweep, run_knowledge_pull, run_matrix_sweep, run_meta_sync, run_pause_drain,
+    run_pause_signal, run_prebuild, run_provision, run_purge_state, run_rebuild_bookkeeping,
+    run_reconcile, run_set_wanted, run_signal, run_start, run_stop, run_stop_for_update, run_swap,
+    run_wanted_pull, run_write_dropin,
 };
 use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
 use crate::job_queue::model::NodeKind;
+use crate::meta_rev::MetaBuild;
 use crate::power::Wanted;
 
 const REV: &str = "2222222222222222222222222222222222222222";
@@ -23,8 +28,13 @@ const REV: &str = "2222222222222222222222222222222222222222";
 struct FakeHost {
     running: bool,
     rev: Option<String>,
-    /// Names of the calls that fail: `kill`, `prebuild`, `swap_update`, `set_paused`.
+    /// Names of the calls that fail (`kill`, `swap_update`, ...). `recorded_build` fails by
+    /// finding nothing and `matrix_ensure_all` by reporting false.
     failing: Vec<&'static str>,
+    /// The sweeps and pulls never finish, so only a deadline ends them.
+    hang: bool,
+    runtime: PathBuf,
+    purge: Vec<PathBuf>,
     calls: Mutex<Vec<String>>,
 }
 
@@ -54,6 +64,15 @@ impl FakeHost {
         self.failing.contains(&call)
     }
 
+    /// Record `line`, then fail when `call` is on the failing list.
+    fn step(&self, call: &str, line: String) -> anyhow::Result<()> {
+        self.record(line);
+        if self.fails(call) {
+            anyhow::bail!("{call} failed");
+        }
+        Ok(())
+    }
+
     fn record(&self, call: String) {
         self.calls.lock().unwrap().push(call);
     }
@@ -114,6 +133,121 @@ impl Host for FakeHost {
         }
         Ok(())
     }
+
+    async fn notify(&self, key: String, summary: String) {
+        self.record(format!("notify {key} | {summary}"));
+    }
+
+    async fn refresh_archetype(&self, name: &str) -> anyhow::Result<()> {
+        self.step("refresh_archetype", format!("refresh_archetype {name}"))
+    }
+
+    async fn prepare_rebuild_dirs(&self, name: &str, _paths: &AgentPaths) -> anyhow::Result<()> {
+        self.step(
+            "prepare_rebuild_dirs",
+            format!("prepare_rebuild_dirs {name}"),
+        )
+    }
+
+    async fn refresh_agent_set(&self) -> anyhow::Result<()> {
+        self.step("refresh_agent_set", "refresh_agent_set".to_owned())
+    }
+
+    async fn resolve_meta(&self, name: &str) -> anyhow::Result<MetaBuild> {
+        self.step("resolve_meta", format!("resolve_meta {name}"))?;
+        Ok(build())
+    }
+
+    fn record_meta(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
+        self.step("record_meta", format!("record_meta {name} {}", build.rev))
+    }
+
+    fn recorded_build(&self, name: &str) -> Option<MetaBuild> {
+        self.record(format!("recorded_build {name}"));
+        (!self.fails("recorded_build")).then(build)
+    }
+
+    fn mark_deployed(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
+        self.step(
+            "mark_deployed",
+            format!("mark_deployed {name} {}", build.rev),
+        )
+    }
+
+    async fn provision(&self, name: &str, _paths: &AgentPaths) -> anyhow::Result<()> {
+        self.step("provision", format!("provision {name}"))
+    }
+
+    async fn create_only(&self, name: &str, rev: &str) -> anyhow::Result<()> {
+        self.step("create_only", format!("create_only {name} {rev}"))
+    }
+
+    async fn write_dropins(
+        &self,
+        name: &str,
+        _hive: &HiveEnv,
+        _paths: &AgentPaths,
+    ) -> anyhow::Result<()> {
+        self.step("write_dropins", format!("write_dropins {name}"))
+    }
+
+    async fn start(&self, name: &str, _hive: &HiveEnv, _paths: &AgentPaths) -> anyhow::Result<()> {
+        self.step("start", format!("start {name}"))
+    }
+
+    fn register_agent(&self, _coord: &Arc<Coordinator>, name: &str) -> anyhow::Result<()> {
+        self.step("register_agent", format!("register_agent {name}"))
+    }
+
+    async fn destroy(&self, name: &str) -> anyhow::Result<()> {
+        self.step("destroy", format!("destroy {name}"))
+    }
+
+    fn runtime_dir(&self, _name: &str) -> PathBuf {
+        self.runtime.clone()
+    }
+
+    async fn delete_state_subvolume(&self, name: &str) -> anyhow::Result<()> {
+        self.step(
+            "delete_state_subvolume",
+            format!("delete_state_subvolume {name}"),
+        )
+    }
+
+    fn purge_dirs(&self, _name: &str) -> Vec<PathBuf> {
+        self.purge.clone()
+    }
+
+    async fn forge_ensure_all(&self) {
+        self.record("forge_ensure_all".to_owned());
+    }
+
+    async fn matrix_ensure_all(&self) -> bool {
+        self.record("matrix_ensure_all".to_owned());
+        if self.hang {
+            std::future::pending::<()>().await;
+        }
+        !self.fails("matrix_ensure_all")
+    }
+
+    async fn knowledge_pull(&self, _coord: &Arc<Coordinator>) -> anyhow::Result<()> {
+        if self.hang {
+            std::future::pending::<()>().await;
+        }
+        self.step("knowledge_pull", "knowledge_pull".to_owned())
+    }
+
+    async fn wanted_pull(&self, _coord: &Arc<Coordinator>) -> anyhow::Result<()> {
+        self.step("wanted_pull", "wanted_pull".to_owned())
+    }
+}
+
+fn build() -> MetaBuild {
+    MetaBuild {
+        rev: REV.to_owned(),
+        hyperhive: "h".to_owned(),
+        config: Some("c".to_owned()),
+    }
 }
 
 fn coordinator() -> (tempfile::TempDir, Arc<Coordinator>) {
@@ -241,11 +375,14 @@ async fn swap_without_a_recorded_rev_never_updates() {
 // --- Stop / StopForUpdate ---------------------------------------------------
 
 #[tokio::test]
-async fn stop_kills_the_container() {
+async fn stop_kills_the_container_then_announces_it() {
     let (_dir, coord) = coordinator();
     let host = FakeHost::up();
     run_stop(&host, &coord, "alice").await.expect("stop ok");
-    assert_eq!(host.calls(), ["kill alice"]);
+    assert_eq!(
+        host.calls(),
+        ["kill alice", "notify killed:alice | agent 'alice' killed"]
+    );
 }
 
 #[tokio::test]
@@ -454,3 +591,315 @@ async fn pause_drain_times_out_ok_and_clears_pending() {
         .expect("a timeout is not an error");
     assert!(!coord.is_pause_pending("alice"));
 }
+
+// --- MetaSync ---------------------------------------------------------------
+
+#[tokio::test]
+async fn meta_sync_prepares_then_resolves_and_records_last() {
+    let host = FakeHost::up();
+    run_meta_sync(&host, "alice").await.expect("ok");
+    assert_eq!(
+        host.calls(),
+        [
+            "refresh_archetype alice".to_owned(),
+            "prepare_rebuild_dirs alice".to_owned(),
+            "refresh_agent_set".to_owned(),
+            "resolve_meta alice".to_owned(),
+            format!("record_meta alice {REV}"),
+        ]
+    );
+}
+
+#[tokio::test]
+async fn meta_sync_keeps_going_when_the_archetype_read_fails() {
+    let host = FakeHost::up().failing("refresh_archetype");
+    run_meta_sync(&host, "alice").await.expect("non-fatal");
+    assert_eq!(host.calls().len(), 5);
+}
+
+#[tokio::test]
+async fn meta_sync_stops_at_the_first_fatal_step() {
+    for (fail, calls) in [
+        ("prepare_rebuild_dirs", 2),
+        ("refresh_agent_set", 3),
+        ("resolve_meta", 4),
+        ("record_meta", 5),
+    ] {
+        let host = FakeHost::up().failing(fail);
+        run_meta_sync(&host, "alice")
+            .await
+            .expect_err("a fatal step fails the node");
+        assert_eq!(host.calls().len(), calls, "{fail}");
+    }
+}
+
+// --- Provision / Create / WriteDropin ----------------------------------------
+
+#[tokio::test]
+async fn provision_and_write_dropin_forward_and_propagate_failure() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up();
+    run_provision(&host, "alice").await.expect("ok");
+    run_write_dropin(&host, &coord, "alice").await.expect("ok");
+    assert_eq!(host.calls(), ["provision alice", "write_dropins alice"]);
+    let host = FakeHost::up().failing("provision").failing("write_dropins");
+    run_provision(&host, "alice").await.expect_err("provision");
+    run_write_dropin(&host, &coord, "alice")
+        .await
+        .expect_err("dropins");
+}
+
+#[tokio::test]
+async fn create_builds_at_the_resolved_rev_then_marks_it_deployed() {
+    let host = FakeHost::up();
+    run_create(&host, "alice").await.expect("ok");
+    assert_eq!(
+        host.calls(),
+        [
+            "resolve_meta alice".to_owned(),
+            format!("create_only alice {REV}"),
+            format!("mark_deployed alice {REV}"),
+        ]
+    );
+}
+
+#[tokio::test]
+async fn a_failed_create_is_never_marked_deployed() {
+    let host = FakeHost::up().failing("create_only");
+    run_create(&host, "alice").await.expect_err("create fails");
+    assert_eq!(host.calls().len(), 2);
+    let host = FakeHost::up().failing("resolve_meta");
+    run_create(&host, "alice").await.expect_err("resolve fails");
+    assert_eq!(host.calls(), ["resolve_meta alice"]);
+}
+
+// --- Start ------------------------------------------------------------------
+
+#[tokio::test]
+async fn start_starts_then_registers_the_listener() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::down();
+    run_start(&host, &coord, "alice").await.expect("ok");
+    assert_eq!(host.calls(), ["start alice", "register_agent alice"]);
+}
+
+#[tokio::test]
+async fn a_failed_start_registers_no_listener() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::down().failing("start");
+    run_start(&host, &coord, "alice").await.expect_err("start");
+    assert_eq!(host.calls(), ["start alice"]);
+}
+
+#[tokio::test]
+async fn a_failed_listener_registration_fails_the_start_node() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::down().failing("register_agent");
+    run_start(&host, &coord, "alice")
+        .await
+        .expect_err("register");
+}
+
+// --- Destroy / Purge ---------------------------------------------------------
+
+#[tokio::test]
+async fn destroy_container_removes_the_runtime_dir() {
+    let (dir, coord) = coordinator();
+    let runtime = dir.path().join("run");
+    std::fs::create_dir_all(runtime.join("sub")).expect("mkdir");
+    let host = FakeHost {
+        runtime: runtime.clone(),
+        ..FakeHost::up()
+    };
+    run_destroy_container(&host, &coord, "alice")
+        .await
+        .expect("ok");
+    assert_eq!(host.calls(), ["destroy alice"]);
+    assert!(!runtime.exists());
+    // A runtime dir that is already gone is fine.
+    run_destroy_container(&host, &coord, "alice")
+        .await
+        .expect("idempotent");
+}
+
+#[tokio::test]
+async fn a_failed_destroy_leaves_the_runtime_dir_alone() {
+    let (dir, coord) = coordinator();
+    let runtime = dir.path().join("run");
+    std::fs::create_dir_all(&runtime).expect("mkdir");
+    let host = FakeHost {
+        runtime: runtime.clone(),
+        ..FakeHost::up().failing("destroy")
+    };
+    run_destroy_container(&host, &coord, "alice")
+        .await
+        .expect_err("destroy fails the node");
+    assert!(runtime.exists());
+}
+
+#[tokio::test]
+async fn purge_removes_every_state_dir_even_when_the_subvolume_delete_fails() {
+    let dir = tempfile::tempdir().expect("tempdir");
+    let (state, applied, absent) = (
+        dir.path().join("state"),
+        dir.path().join("applied"),
+        dir.path().join("absent"),
+    );
+    for d in [&state, &applied] {
+        std::fs::create_dir_all(d.join("deep")).expect("mkdir");
+    }
+    let host = FakeHost {
+        purge: vec![state.clone(), absent, applied.clone()],
+        ..FakeHost::up().failing("delete_state_subvolume")
+    };
+    run_purge_state(&host, "alice").await;
+    assert_eq!(host.calls(), ["delete_state_subvolume alice"]);
+    assert!(!state.exists() && !applied.exists());
+}
+
+#[tokio::test]
+async fn destroy_bookkeeping_drops_the_power_row_and_announces() {
+    let (_dir, coord) = coordinator();
+    coord.power.set("alice", Wanted::Offline).expect("set");
+    let host = FakeHost::up();
+    run_destroy_bookkeeping(&host, &coord, "alice").await;
+    assert_eq!(
+        host.calls(),
+        [
+            "refresh_agent_set",
+            "notify destroyed:alice | agent 'alice' destroyed"
+        ]
+    );
+    // The Offline row is gone, so a fresh seed from "running" gives Up.
+    assert_eq!(
+        coord.power.get_or_seed("alice", true).expect("seed"),
+        Wanted::Up
+    );
+}
+
+#[tokio::test]
+async fn destroy_bookkeeping_continues_past_a_failed_refresh() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up().failing("refresh_agent_set");
+    run_destroy_bookkeeping(&host, &coord, "alice").await;
+    assert_eq!(host.calls().len(), 2);
+}
+
+// --- RebuildBookkeeping / EmitRebuilt ----------------------------------------
+
+#[tokio::test]
+async fn rebuild_bookkeeping_marks_the_recorded_build_deployed() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up();
+    run_rebuild_bookkeeping(&host, &coord, "alice")
+        .await
+        .expect("ok");
+    assert_eq!(
+        host.calls(),
+        [
+            "recorded_build alice".to_owned(),
+            format!("mark_deployed alice {REV}")
+        ]
+    );
+}
+
+#[tokio::test]
+async fn rebuild_bookkeeping_without_a_recording_marks_nothing() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up().failing("recorded_build");
+    run_rebuild_bookkeeping(&host, &coord, "alice")
+        .await
+        .expect("ok");
+    assert_eq!(host.calls(), ["recorded_build alice"]);
+}
+
+#[tokio::test]
+async fn a_failed_marker_write_does_not_fail_the_bookkeeping_node() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up().failing("mark_deployed");
+    run_rebuild_bookkeeping(&host, &coord, "alice")
+        .await
+        .expect("warn only");
+}
+
+#[tokio::test]
+async fn emit_rebuilt_reports_success_and_failure_under_one_key() {
+    let (_dir, coord) = coordinator();
+    let host = FakeHost::up();
+    run_emit_rebuilt(&host, &coord, "alice", None, true).await;
+    run_emit_rebuilt(&host, &coord, "alice", None, false).await;
+    assert_eq!(
+        host.calls(),
+        [
+            "notify rebuilt:alice | agent 'alice' rebuilt",
+            "notify rebuilt:alice | agent 'alice' rebuild FAILED: unknown error",
+        ]
+    );
+}
+
+// --- Sweeps and pulls ---------------------------------------------------------
+
+#[tokio::test]
+async fn forge_sweep_runs_once_and_never_fails() {
+    let host = FakeHost::up();
+    run_forge_sweep(&host).await.expect("ok");
+    assert_eq!(host.calls(), ["forge_ensure_all"]);
+}
+
+#[tokio::test]
+async fn matrix_sweep_fails_the_node_when_an_agent_failed_sync() {
+    run_matrix_sweep(&FakeHost::up(), Duration::from_secs(5))
+        .await
+        .expect("ok");
+    let err = run_matrix_sweep(
+        &FakeHost::up().failing("matrix_ensure_all"),
+        Duration::from_secs(5),
+    )
+    .await
+    .expect_err("failed sync");
+    assert!(format!("{err:#}").contains("failed sync"), "{err:#}");
+}
+
+#[tokio::test]
+async fn matrix_sweep_past_its_deadline_fails_instead_of_hanging() {
+    let host = FakeHost {
+        hang: true,
+        ..FakeHost::up()
+    };
+    run_matrix_sweep(&host, Duration::from_millis(50))
+        .await
+        .expect_err("deadline");
+}
+
+#[tokio::test]
+async fn knowledge_pull_passes_the_result_through_and_times_out() {
+    let (_dir, coord) = coordinator();
+    let secs = Duration::from_secs(5);
+    run_knowledge_pull(&FakeHost::up(), &coord, secs)
+        .await
+        .expect("ok");
+    let err = run_knowledge_pull(&FakeHost::up().failing("knowledge_pull"), &coord, secs)
+        .await
+        .expect_err("pull fails");
+    assert!(
+        format!("{err:#}").contains("knowledge_pull failed"),
+        "{err:#}"
+    );
+    let hang = FakeHost {
+        hang: true,
+        ..FakeHost::up()
+    };
+    let err = run_knowledge_pull(&hang, &coord, Duration::from_millis(50))
+        .await
+        .expect_err("deadline");
+    assert!(format!("{err:#}").contains("timed out"), "{err:#}");
+}
+
+#[tokio::test]
+async fn wanted_pull_passes_the_result_through() {
+    let (_dir, coord) = coordinator();
+    run_wanted_pull(&FakeHost::up(), &coord).await.expect("ok");
+    run_wanted_pull(&FakeHost::up().failing("wanted_pull"), &coord)
+        .await
+        .expect_err("read failure reaches the node");
+}