hyperhive/swarmctl/src/setup.rs

//! `swarmctl setup status` — where the first-run ceremony of
//! `docs/1-getting-started/setup.md` stands.
//!
//! Read-only, and needs no token: every step is judged from something an
//! operator could see without one. Observing and judging are separate on
//! purpose. [`observe`] runs the commands and calls and records what came
//! back in [`Observed`]; [`classify`] turns that into one [`Step`] per
//! ceremony step and is pure, so every step's outcome is tested against
//! hand-built observations.
//!
//! A dependency that cannot be reached makes its steps `unknown (<why>)`,
//! never an error: the verb exists to be run on a half-built host.

use std::path::Path;
use std::process::Command;

use anyhow::{Context as _, Result};
use hive_types::Permission;
use serde::Deserialize;
use swarm_controller_api::permissions::Permissions;

use crate::agent::send;
use crate::remote::Target;

/// The manager agent the ceremony creates.
const RUTH: &str = "ruth";
/// The SSO group the swarm UI and other operator surfaces gate on.
const ADMINS_GROUP: &str = "admins";
const GRANTER_UNIT: &str = "swarm-bao-granter-role.service";
/// What `swarm-bao-granter-role` logs when it has written the granter's policy.
const GRANTER_MARKER: &str = "Uploaded policy: bao-granter";
/// Waits for authelia, which waits for the policies written in step 4, so it
/// is not a signal for step 4 itself.
const POLICY_UNIT_SKIPPED: &str = "operator-viewer";

/// What one step's check found.
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum Status {
    /// The detail may be empty.
    Done(String),
    NotDone(String),
    /// The check could not run, or its answer could not be read.
    Unknown(String),
    /// Nothing the host or the controller exposes shows this step.
    Unobservable(String),
}

#[derive(Debug)]
pub(crate) struct Step {
    pub(crate) number: u8,
    pub(crate) title: &'static str,
    pub(crate) status: Status,
    /// What to do about it: a doc section or a verb.
    pub(crate) hint: &'static str,
    /// A step the ceremony allows skipping; it never fails the exit code.
    pub(crate) optional: bool,
}

/// `bao status`, read.
#[derive(Debug)]
pub(crate) struct Store {
    pub(crate) initialized: bool,
    pub(crate) sealed: bool,
    /// The seal type: `shamir` needs a person, the others open themselves.
    pub(crate) seal: String,
}

#[derive(Debug)]
pub(crate) struct Granter {
    /// systemd's `ActiveState` of `swarm-bao-granter-role`.
    pub(crate) state: String,
    /// The unit's journal holds [`GRANTER_MARKER`].
    pub(crate) uploaded: bool,
}

#[derive(Debug)]
pub(crate) struct Unit {
    pub(crate) name: String,
    pub(crate) state: String,
}

/// One row of `GET /api/agents/status`, the two fields read.
#[derive(Debug, Deserialize)]
pub(crate) struct AgentRow {
    pub(crate) name: String,
    #[serde(default)]
    pub(crate) wanted: Option<String>,
}

#[derive(Debug)]
pub(crate) struct Agents {
    pub(crate) rows: Vec<AgentRow>,
    /// Whether the swarm grants ruth `manage_root_agent`; `None` while she
    /// has no row.
    pub(crate) ruth_manages_root: Option<Result<bool, String>>,
}

/// Everything [`classify`] judges. Each field is the whole answer of one
/// check, or the reason the check failed.
#[derive(Debug)]
pub(crate) struct Observed {
    pub(crate) store: Result<Store, String>,
    pub(crate) granter: Result<Granter, String>,
    pub(crate) policy_units: Result<Vec<Unit>, String>,
    /// Whether the SSO store holds an enabled member of [`ADMINS_GROUP`].
    pub(crate) admin_user: Result<bool, String>,
    pub(crate) agents: Result<Agents, String>,
}
/// Each step's title, what to do about it, and whether it is optional, in
/// ceremony order.
const STEPS: [(&str, &str, bool); 10] = [
    (
        "secret store initialised",
        "setup.md § 1: `sudo bao operator init`, on the host running swarm-bao",
        false,
    ),
    (
        "secret store unsealed",
        "setup.md § 1: `bao operator unseal`",
        false,
    ),
    (
        "granter bootstrapped",
        "setup.md § 1: the bao-bootstrap block, then `systemctl restart swarm-bao-granter-role`",
        false,
    ),
    (
        "granting units active",
        "setup.md § 1: `systemctl reset-failed` then `systemctl restart` the `swarm-bao-*-policy` units and `swarm-bao-agent-pki`",
        false,
    ),
    (
        "store unsealed after the granting units",
        "setup.md § 1: `bao operator unseal` again after the store restarted",
        false,
    ),
    (
        "SSO account in the admins group",
        "setup.md § 2: `swarmctl user add <name> --email <address> --group admins`",
        false,
    ),
    (
        "first swarm UI and forge login",
        "setup.md § 3: sign in at https://<swarm.domain>/, then to the forge",
        false,
    ),
    (
        "forge site admin",
        "setup.md § 3: `swarmctl forge make-admin <name>`",
        false,
    ),
    (
        "ruth created and up",
        "setup.md § 4: create ruth from the swarm UI's clippy form, then set her `up` on /agents",
        false,
    ),
    (
        "another agent",
        "setup.md § 5: `swarmctl agent create <name> --hive <hive>`",
        true,
    ),
];

/// Judge `observed` step by step, in ceremony order.
pub(crate) fn classify(observed: &Observed) -> Vec<Step> {
    let unsealed = unsealed_status(&observed.store);
    let policy_units = policy_units_status(&observed.policy_units);
    let granter = granter_status(&observed.granter, &policy_units);
    let reunsealed = match (&unsealed, &policy_units) {
        (Status::Done(_), Status::Done(_)) => Status::Done(String::new()),
        (Status::Unknown(e), _) | (_, Status::Unknown(e)) => Status::Unknown(e.clone()),
        (Status::NotDone(why), _) => Status::NotDone(why.clone()),
        _ => Status::NotDone("waits for step 4".to_owned()),
    };
    let statuses = [
        initialized_status(&observed.store),
        unsealed,
        granter,
        policy_units,
        reunsealed,
        admin_user_status(&observed.admin_user),
        Status::Unobservable("a browser login leaves nothing to read".to_owned()),
        Status::Unobservable(
            "the controller has no read route for it; `forge make-admin` is safe to re-run"
                .to_owned(),
        ),
        ruth_status(&observed.agents),
        others_status(&observed.agents),
    ];
    STEPS
        .into_iter()
        .zip(statuses)
        .zip(1..)
        .map(|(((title, hint, optional), status), number)| Step {
            number,
            title,
            status,
            hint,
            optional,
        })
        .collect()
}

fn initialized_status(store: &Result<Store, String>) -> Status {
    match store {
        Ok(s) if s.initialized => Status::Done(String::new()),
        Ok(_) => Status::NotDone("the store is not initialised".to_owned()),
        Err(e) => Status::Unknown(e.clone()),
    }
}

fn unsealed_status(store: &Result<Store, String>) -> Status {
    match store {
        Ok(s) if !s.initialized => Status::NotDone("not initialised yet".to_owned()),
        Ok(s) if !s.sealed => Status::Done(String::new()),
        Ok(s) if s.seal == "shamir" => Status::NotDone("the store is sealed".to_owned()),
        Ok(s) => Status::NotDone(format!(
            "the store is sealed and its {} seal did not open it",
            s.seal
        )),
        Err(e) => Status::Unknown(e.clone()),
    }
}

fn admin_user_status(admin_user: &Result<bool, String>) -> Status {
    match admin_user {
        Ok(true) => Status::Done(String::new()),
        Ok(false) => Status::NotDone(format!(
            "no enabled user in the {ADMINS_GROUP:?} group; the placeholder is disabled"
        )),
        Err(e) => Status::Unknown(e.clone()),
    }
}

fn policy_units_status(units: &Result<Vec<Unit>, String>) -> Status {
    match units {
        Err(e) => Status::Unknown(e.clone()),
        Ok(units) => {
            let idle: Vec<String> = units
                .iter()
                .filter(|u| u.state != "active")
                .map(|u| format!("{} is {}", u.name, u.state))
                .collect();
            if idle.is_empty() {
                Status::Done(format!("{} units", units.len()))
            } else {
                Status::NotDone(idle.join(", "))
            }
        }
    }
}

/// The granter unit deletes its token when it succeeds and is skipped when
/// the token is absent, so after a reboot it is `inactive` whether or not it
/// ever ran. An active granting unit logs in through the granter, so that
/// also proves the granter exists.
fn granter_status(granter: &Result<Granter, String>, policy_units: &Status) -> Status {
    if matches!(policy_units, Status::Done(_)) {
        return Status::Done("the granting units log in through it".to_owned());
    }
    match granter {
        Err(e) => Status::Unknown(e.clone()),
        Ok(g) if g.state == "active" => Status::Done(String::new()),
        Ok(g) if g.uploaded => Status::Done("its journal shows the upload".to_owned()),
        Ok(g) if g.state == "failed" => {
            Status::NotDone("swarm-bao-granter-role failed; see its journal".to_owned())
        }
        Ok(_) => Status::NotDone("swarm-bao-granter-role has not uploaded the policy".to_owned()),
    }
}

fn ruth_status(agents: &Result<Agents, String>) -> Status {
    let agents = match agents {
        Ok(a) => a,
        Err(e) => return Status::Unknown(e.clone()),
    };
    let Some(ruth) = agents.rows.iter().find(|r| r.name == RUTH) else {
        return Status::NotDone(format!("{RUTH} is not in the swarm's roster"));
    };
    match &agents.ruth_manages_root {
        Some(Err(e)) => return Status::Unknown(format!("{RUTH}'s permissions: {e}")),
        Some(Ok(false)) | None => {
            return Status::NotDone(format!("{RUTH} lacks manage_root_agent"));
        }
        Some(Ok(true)) => {}
    }
    match ruth.wanted.as_deref() {
        Some("up") => Status::Done(String::new()),
        Some(state) => Status::NotDone(format!("{RUTH} is declared {state}")),
        None => Status::NotDone(format!("{RUTH} has no declared state")),
    }
}

fn others_status(agents: &Result<Agents, String>) -> Status {
    match agents {
        Err(e) => Status::Unknown(e.clone()),
        Ok(a) => {
            let others = a.rows.iter().filter(|r| r.name != RUTH).count();
            if others > 0 {
                Status::Done(format!("{others} besides {RUTH}"))
            } else {
                Status::NotDone(format!("no agent besides {RUTH}"))
            }
        }
    }
}

/// Whether the ceremony is as complete as it can be seen: every step is done,
/// or cannot be observed, or is optional. A step whose check failed counts
/// against it.
pub(crate) fn complete(steps: &[Step]) -> bool {
    steps
        .iter()
        .all(|s| s.optional || matches!(s.status, Status::Done(_) | Status::Unobservable(_)))
}

/// One line per step, then what to do next.
pub(crate) fn render(steps: &[Step]) -> String {
    use std::fmt::Write as _;

    let mut out = String::new();
    for step in steps {
        let status = match &step.status {
            Status::Done(d) if d.is_empty() => "done".to_owned(),
            Status::Done(d) => format!("done ({d})"),
            Status::NotDone(why) => format!("not done ({why})"),
            Status::Unknown(why) => format!("unknown ({why})"),
            Status::Unobservable(why) => format!("unknown (not observable: {why})"),
        };
        let optional = if step.optional { " [optional]" } else { "" };
        let _ = writeln!(
            out,
            "{:>2}  {}{optional}: {status}",
            step.number, step.title
        );
    }
    if let Some(next) = steps
        .iter()
        .find(|s| !s.optional && matches!(s.status, Status::NotDone(_)))
    {
        let _ = writeln!(out, "next: step {}, {}", next.number, next.hint);
    } else if let Some(blind) = steps
        .iter()
        .find(|s| !s.optional && matches!(s.status, Status::Unknown(_)))
    {
        let _ = writeln!(
            out,
            "next: step {} could not be checked; fix the cause above and run this again",
            blind.number
        );
    }
    out
}

/// Run `swarmctl setup status`. Returns whether the ceremony is [`complete`].
///
/// `target` and `users_file` arrive as the results of resolving them, so a
/// host that lacks either still reports the steps that need neither.
pub(crate) fn status(target: Result<Target>, users_file: Result<std::path::PathBuf>) -> bool {
    let observed = observe(target, users_file);
    let steps = classify(&observed);
    print!("{}", render(&steps));
    complete(&steps)
}

fn observe(target: Result<Target>, users_file: Result<std::path::PathBuf>) -> Observed {
    let store = bao_status();
    let granter = granter();
    let policy_units = policy_units();
    let admin_user = users_file
        .map_err(|e| unset_reason(&e))
        .and_then(|path| has_admin_user(&path).map_err(|e| format!("{e:#}")));
    let agents = target
        .map_err(|e| unset_reason(&e))
        .and_then(|t| agents(&t).map_err(|e| format!("{e:#}")));
    Observed {
        store,
        granter,
        policy_units,
        admin_user,
        agents,
    }
}

/// The part of a resolution error before its first em dash, which names the
/// unset variable; the rest explains the deployment.
fn unset_reason(e: &anyhow::Error) -> String {
    let full = format!("{e:#}");
    full.split(" — ").next().unwrap_or(&full).to_owned()
}

struct Output {
    code: Option<i32>,
    stdout: String,
    stderr: String,
}

fn run(command: &mut Command) -> Result<Output, String> {
    let program = command.get_program().to_string_lossy().into_owned();
    let out = command.output().map_err(|e| match e.kind() {
        std::io::ErrorKind::NotFound => {
            format!("{program} is not on PATH; run this on the host that runs swarm-bao")
        }
        _ => format!("running {program}: {e}"),
    })?;
    Ok(Output {
        code: out.status.code(),
        stdout: String::from_utf8_lossy(&out.stdout).into_owned(),
        stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
    })
}

/// `bao status`: the wrapper on `PATH` carries the swarm's address, CA and
/// client certificate, and the call needs no token. A client timeout is set
/// when the environment has none, so an unreachable store answers.
fn bao_status() -> Result<Store, String> {
    let mut command = Command::new("bao");
    command.args(["status", "-format=json"]);
    if std::env::var_os("BAO_CLIENT_TIMEOUT").is_none() {
        command.env("BAO_CLIENT_TIMEOUT", "10s");
    }
    let out = run(&mut command)?;
    parse_bao_status(out.code, &out.stdout, &out.stderr)
}

/// Exit 0 is unsealed, 2 is sealed (an uninitialised store is sealed too, and
/// its JSON says `"initialized": false`), anything else is an error.
fn parse_bao_status(code: Option<i32>, stdout: &str, stderr: &str) -> Result<Store, String> {
    #[derive(Deserialize)]
    struct Status {
        initialized: bool,
        sealed: bool,
        #[serde(rename = "type", default)]
        seal: String,
    }
    match code {
        Some(0 | 2) => {
            let s: Status = serde_json::from_str(stdout)
                .map_err(|e| format!("bao status printed JSON this cannot read: {e}"))?;
            Ok(Store {
                initialized: s.initialized,
                sealed: s.sealed,
                seal: s.seal,
            })
        }
        other => Err(format!(
            "bao status {}: {}",
            other.map_or_else(|| "was killed".to_owned(), |c| format!("exited {c}")),
            stderr.lines().next().unwrap_or("no message")
        )),
    }
}

fn granter() -> Result<Granter, String> {
    let shown = run(Command::new("systemctl").args([
        "show",
        GRANTER_UNIT,
        "-p",
        "LoadState",
        "-p",
        "ActiveState",
    ]))?;
    let state = parse_unit_state(&shown.stdout, GRANTER_UNIT)?;
    // An unreadable journal counts as no upload: the unit's state decides.
    let uploaded = run(Command::new("journalctl").args(["-u", GRANTER_UNIT, "--no-pager", "-q"]))
        .is_ok_and(|j| j.stdout.contains(GRANTER_MARKER));
    Ok(Granter { state, uploaded })
}

/// `ActiveState` out of `systemctl show -p LoadState -p ActiveState`.
fn parse_unit_state(shown: &str, unit: &str) -> Result<String, String> {
    let field = |name: &str| {
        shown
            .lines()
            .find_map(|l| l.strip_prefix(name).and_then(|v| v.strip_prefix('=')))
    };
    match (field("LoadState"), field("ActiveState")) {
        (Some("not-found"), _) => Err(format!(
            "{unit} does not exist here; run this on the host that runs swarm-bao"
        )),
        (_, Some(state)) => Ok(state.to_owned()),
        _ => Err(format!("systemctl show {unit} printed no ActiveState")),
    }
}

fn policy_units() -> Result<Vec<Unit>, String> {
    let listed = run(Command::new("systemctl").args([
        "list-unit-files",
        "swarm-bao-*-policy.service",
        "swarm-bao-agent-pki.service",
        "--no-legend",
    ]))?;
    let names = parse_unit_files(&listed.stdout);
    if names.is_empty() {
        return Err(
            "no swarm-bao-*-policy units here; run this on the host that runs swarm-bao".to_owned(),
        );
    }
    // `is-active` exits non-zero when any unit is not active, so its exit
    // status says nothing; it prints one state per unit, in order.
    let active = run(Command::new("systemctl").arg("is-active").args(&names))?;
    let states: Vec<&str> = active.stdout.lines().collect();
    if states.len() != names.len() {
        return Err(format!(
            "systemctl is-active answered {} states for {} units",
            states.len(),
            names.len()
        ));
    }
    Ok(names
        .into_iter()
        .zip(states)
        .map(|(name, state)| Unit {
            name,
            state: state.to_owned(),
        })
        .collect())
}

/// The unit names in `systemctl list-unit-files --no-legend`, without the one
/// that waits on authelia.
fn parse_unit_files(listed: &str) -> Vec<String> {
    listed
        .lines()
        .filter_map(|l| l.split_whitespace().next())
        .filter(|name| name.ends_with(".service") && !name.contains(POLICY_UNIT_SKIPPED))
        .map(str::to_owned)
        .collect()
}

/// Whether the users database holds an enabled member of [`ADMINS_GROUP`].
fn has_admin_user(users_file: &Path) -> Result<bool> {
    if !users_file.exists() {
        anyhow::bail!(
            "{} does not exist; run this on the host that runs authelia",
            users_file.display()
        );
    }
    let store = crate::load_store(users_file)?;
    Ok(store.users.values().any(|u| {
        u.groups.iter().any(|g| g == ADMINS_GROUP)
            && !matches!(
                u.extra.get("disabled"),
                Some(serde_norway::Value::Bool(true))
            )
    }))
}

fn agents(target: &Target) -> Result<Agents> {
    let rt = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .context("starting a tokio runtime for the controller request")?;
    let rows: Vec<AgentRow> = rt.block_on(send(
        target,
        hyper::Method::GET,
        "/api/agents/status",
        None::<&()>,
        "agents-status",
    ))?;
    let ruth_manages_root = rows.iter().any(|r| r.name == RUTH).then(|| {
        rt.block_on(send::<(), Permissions>(
            target,
            hyper::Method::GET,
            &format!("/api/agents/{RUTH}/permissions"),
            None,
            "permissions-get",
        ))
        .map(|p| {
            p.permissions
                .is_some_and(|granted| granted.contains(&Permission::ManageRootAgent))
        })
        .map_err(|e| format!("{e:#}"))
    });
    Ok(Agents {
        rows,
        ruth_manages_root,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    fn store(initialized: bool, sealed: bool, seal: &str) -> Store {
        Store {
            initialized,
            sealed,
            seal: seal.to_owned(),
        }
    }

    fn units(states: &[&str]) -> Vec<Unit> {
        states
            .iter()
            .enumerate()
            .map(|(i, s)| Unit {
                name: format!("swarm-bao-u{i}-policy.service"),
                state: (*s).to_owned(),
            })
            .collect()
    }

    fn row(name: &str, wanted: Option<&str>) -> AgentRow {
        AgentRow {
            name: name.to_owned(),
            wanted: wanted.map(str::to_owned),
        }
    }

    fn agents(rows: Vec<AgentRow>, ruth_root: Option<Result<bool, String>>) -> Agents {
        Agents {
            rows,
            ruth_manages_root: ruth_root,
        }
    }

    /// Everything done, as a finished ceremony looks.
    fn finished() -> Observed {
        Observed {
            store: Ok(store(true, false, "shamir")),
            granter: Ok(Granter {
                state: "active".to_owned(),
                uploaded: true,
            }),
            policy_units: Ok(units(&["active", "active"])),
            admin_user: Ok(true),
            agents: Ok(agents(
                vec![row("ruth", Some("up")), row("iris", Some("paused"))],
                Some(Ok(true)),
            )),
        }
    }

    fn status_of(observed: &Observed, number: u8) -> Status {
        classify(observed)
            .into_iter()
            .find(|s| s.number == number)
            .unwrap()
            .status
    }

    fn kind(status: &Status) -> &'static str {
        match status {
            Status::Done(_) => "done",
            Status::NotDone(_) => "not done",
            Status::Unknown(_) => "unknown",
            Status::Unobservable(_) => "unobservable",
        }
    }

    fn kinds(observed: &Observed) -> Vec<&'static str> {
        classify(observed).iter().map(|s| kind(&s.status)).collect()
    }

    #[test]
    fn a_finished_ceremony_reads_done_and_exits_clean() {
        let steps = classify(&finished());
        assert_eq!(
            kinds(&finished()),
            [
                "done",
                "done",
                "done",
                "done",
                "done",
                "done",
                "unobservable",
                "unobservable",
                "done",
                "done"
            ]
        );
        assert!(complete(&steps));
    }

    #[test]
    fn an_uninitialised_store_fails_steps_one_two_and_five() {
        let mut o = finished();
        o.store = Ok(store(false, true, "shamir"));
        o.policy_units = Ok(units(&["inactive"]));
        o.granter = Ok(Granter {
            state: "inactive".to_owned(),
            uploaded: false,
        });
        let k = kinds(&o);
        assert_eq!(&k[..5], ["not done"; 5]);
        assert!(!complete(&classify(&o)));
    }

    #[test]
    fn a_sealed_store_is_initialised_but_not_unsealed() {
        let mut o = finished();
        o.store = Ok(store(true, true, "shamir"));
        assert_eq!(kind(&status_of(&o, 1)), "done");
        assert_eq!(kind(&status_of(&o, 2)), "not done");
        // The granting units are active, so only the second unseal is open.
        assert_eq!(kind(&status_of(&o, 4)), "done");
        assert_eq!(kind(&status_of(&o, 5)), "not done");
    }

    #[test]
    fn a_sealed_auto_unseal_store_names_its_seal() {
        let mut o = finished();
        o.store = Ok(store(true, true, "pkcs11"));
        assert!(matches!(status_of(&o, 2), Status::NotDone(why) if why.contains("pkcs11")));
    }

    #[test]
    fn step_five_waits_for_the_granting_units() {
        let mut o = finished();
        o.policy_units = Ok(units(&["active", "failed"]));
        assert!(matches!(status_of(&o, 5), Status::NotDone(why) if why.contains("step 4")));
    }

    #[test]
    fn the_granter_is_done_on_any_one_of_its_three_signals() {
        let mut o = finished();
        o.policy_units = Ok(units(&["failed"]));
        let granter = |state: &str, uploaded| {
            Ok(Granter {
                state: state.to_owned(),
                uploaded,
            })
        };
        o.granter = granter("active", false);
        assert_eq!(kind(&status_of(&o, 3)), "done");
        o.granter = granter("inactive", true);
        assert_eq!(kind(&status_of(&o, 3)), "done");
        // The control: neither signal, and the units not active either.
        o.granter = granter("inactive", false);
        assert_eq!(kind(&status_of(&o, 3)), "not done");
        o.granter = granter("failed", false);
        assert!(matches!(status_of(&o, 3), Status::NotDone(why) if why.contains("failed")));
        // Active granting units prove the granter even with no trace of its unit.
        o.policy_units = Ok(units(&["active"]));
        assert_eq!(kind(&status_of(&o, 3)), "done");
    }

    #[test]
    fn one_inactive_policy_unit_fails_step_four_and_names_it() {
        let mut o = finished();
        o.policy_units = Ok(units(&["active", "failed", "active"]));
        assert!(
            matches!(status_of(&o, 4), Status::NotDone(why) if why.contains("u1-policy") && why.contains("failed") && !why.contains("u0"))
        );
    }

    #[test]
    fn a_missing_admin_account_fails_step_six() {
        let mut o = finished();
        o.admin_user = Ok(false);
        assert_eq!(kind(&status_of(&o, 6)), "not done");
    }

    #[test]
    fn ruth_is_done_only_when_present_permitted_and_up() {
        let mut o = finished();
        o.agents = Ok(agents(vec![], None));
        assert!(matches!(status_of(&o, 9), Status::NotDone(why) if why.contains("roster")));
        o.agents = Ok(agents(vec![row("ruth", Some("paused"))], Some(Ok(true))));
        assert!(matches!(status_of(&o, 9), Status::NotDone(why) if why.contains("paused")));
        o.agents = Ok(agents(vec![row("ruth", None)], Some(Ok(true))));
        assert!(matches!(status_of(&o, 9), Status::NotDone(why) if why.contains("no declared")));
        o.agents = Ok(agents(vec![row("ruth", Some("up"))], Some(Ok(false))));
        assert!(
            matches!(status_of(&o, 9), Status::NotDone(why) if why.contains("manage_root_agent"))
        );
        o.agents = Ok(agents(
            vec![row("ruth", Some("up"))],
            Some(Err("503".to_owned())),
        ));
        assert_eq!(kind(&status_of(&o, 9)), "unknown");
        // The control for the four cases above: the same row, permitted, is done.
        o.agents = Ok(agents(vec![row("ruth", Some("up"))], Some(Ok(true))));
        assert_eq!(kind(&status_of(&o, 9)), "done");
    }

    #[test]
    fn another_agent_is_optional_and_never_counts_ruth() {
        let mut o = finished();
        o.agents = Ok(agents(vec![row("ruth", Some("up"))], Some(Ok(true))));
        assert_eq!(kind(&status_of(&o, 10)), "not done");
        // Optional, so the exit code stays clean.
        assert!(complete(&classify(&o)));
        o.agents = Ok(agents(vec![row("iris", None)], None));
        assert_eq!(kind(&status_of(&o, 10)), "done");
    }

    #[test]
    fn an_unreachable_dependency_makes_its_steps_unknown_and_fails_the_exit() {
        fn down<T>(what: &str) -> Result<T, String> {
            Err(what.to_owned())
        }
        let o = Observed {
            store: down("bao is not on PATH"),
            granter: down("no unit"),
            policy_units: down("no units"),
            admin_user: down("no users file"),
            agents: down("no socket"),
        };
        assert_eq!(
            kinds(&o),
            [
                "unknown",
                "unknown",
                "unknown",
                "unknown",
                "unknown",
                "unknown",
                "unobservable",
                "unobservable",
                "unknown",
                "unknown"
            ]
        );
        assert!(!complete(&classify(&o)));
        let text = render(&classify(&o));
        assert!(text.contains("unknown (bao is not on PATH)"));
        assert!(text.contains("next: step 1 could not be checked"));
    }

    #[test]
    fn the_report_names_the_next_step() {
        let mut o = finished();
        o.admin_user = Ok(false);
        let text = render(&classify(&o));
        assert!(text.contains(" 6  SSO account in the admins group: not done"));
        assert!(text.contains("next: step 6, setup.md § 2"));
        assert!(text.contains("unknown (not observable:"));
        // Nothing left to do, so no pointer.
        assert!(!render(&classify(&finished())).contains("next:"));
    }

    #[test]
    fn bao_status_maps_its_exit_codes() {
        let json =
            |init, sealed| format!(r#"{{"type":"shamir","initialized":{init},"sealed":{sealed}}}"#);
        let unsealed = parse_bao_status(Some(0), &json(true, false), "").unwrap();
        assert!(unsealed.initialized && !unsealed.sealed && unsealed.seal == "shamir");
        let sealed = parse_bao_status(Some(2), &json(true, true), "").unwrap();
        assert!(sealed.initialized && sealed.sealed);
        let fresh = parse_bao_status(Some(2), &json(false, true), "").unwrap();
        assert!(!fresh.initialized && fresh.sealed);
        let down =
            parse_bao_status(Some(1), "", "Error checking seal status: refused\nmore").unwrap_err();
        assert_eq!(
            down,
            "bao status exited 1: Error checking seal status: refused"
        );
        assert!(
            parse_bao_status(None, "", "")
                .unwrap_err()
                .contains("killed")
        );
        assert!(parse_bao_status(Some(2), "not json", "").is_err());
    }

    #[test]
    fn unit_files_drop_the_viewer_and_keep_the_pki_unit() {
        let listed = "swarm-bao-controller-policy.service enabled enabled\n\
                      swarm-bao-operator-viewer-policy.service enabled enabled\n\
                      swarm-bao-agent-pki.service enabled enabled\n";
        assert_eq!(
            parse_unit_files(listed),
            [
                "swarm-bao-controller-policy.service",
                "swarm-bao-agent-pki.service"
            ]
        );
        assert!(parse_unit_files("").is_empty());
    }

    #[test]
    fn unit_state_reads_active_state_and_refuses_a_missing_unit() {
        let shown = "LoadState=loaded\nActiveState=inactive\n";
        assert_eq!(parse_unit_state(shown, "u.service").unwrap(), "inactive");
        let missing = "LoadState=not-found\nActiveState=inactive\n";
        assert!(
            parse_unit_state(missing, "u.service")
                .unwrap_err()
                .contains("does not exist")
        );
        assert!(parse_unit_state("", "u.service").is_err());
    }

    #[test]
    fn admin_membership_ignores_disabled_and_ungrouped_users() {
        let dir = std::env::temp_dir().join(format!("swarmctl-setup-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let file = dir.join("users.yml");
        let write = |body: &str| std::fs::write(&file, body).unwrap();
        let user = |extra: &str, groups: &str| {
            format!("  u:\n    displayname: U\n    password: x\n{groups}{extra}")
        };
        let admins = "    groups:\n      - admins\n";
        write(&format!("users:\n{}", user("    disabled: true\n", admins)));
        assert!(!has_admin_user(&file).unwrap());
        write(&format!("users:\n{}", user("", "")));
        assert!(!has_admin_user(&file).unwrap());
        // The control: the same user, enabled and in the group, is found.
        write(&format!("users:\n{}", user("", admins)));
        assert!(has_admin_user(&file).unwrap());
        assert!(
            has_admin_user(&dir.join("absent.yml"))
                .unwrap_err()
                .to_string()
                .contains("does not exist")
        );
        std::fs::remove_dir_all(&dir).unwrap();
    }
}