hyperhive/swarm-matrix-ctl/src/appservice.rs

//! `swarm-matrix-ctl appservice` — the **swarm's** own appservice
//! registration. The store holds it; this container renders a copy the
//! homeserver starts from onto its own tmpfs, and keeps none on disk.
//!
//! Two verbs, both reading the store:
//!
//! - [`render`] runs before tuwunel. tuwunel loads the file it writes through
//!   `LoadCredential`, and a missing credential source fails the homeserver's
//!   start, so a store that does not answer fails this unit and keeps the
//!   homeserver down until it does. It mints a pair only when the store
//!   answers and holds none.
//! - [`sync`] runs beside the homeserver and follows the store: it rewrites
//!   the file when the store's tokens differ, restarts the homeserver
//!   whenever the registration is newer than the homeserver's last start, and
//!   publishes the `as_token` for the controller.
//!
//! Where the store holds none, both first store the tokens an earlier version
//! kept in the legacy state dir (see [`resolve`]), so a host that has run for
//! months keeps its value. The files stay, unused.
//!
//! This registration's sender is promoted to homeserver admin at boot
//! (`nix/host-modules/hive-matrix.nix`), which is why its `as_token` goes to
//! `swarm_secret_client::matrix::swarm_appservice_token_path` for
//! `swarm-controller` — a path no hive's policy reaches — and to nowhere else.

use std::io::Write as _;
use std::os::unix::fs::OpenOptionsExt as _;
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, bail};
use swarm_secret_client::{
    SecretStore,
    client::{DEFAULT_CERT_MOUNT, Settings},
    matrix::{self, Registration},
};

/// Where the homeserver's copy is rendered: a runtime directory on this
/// container's tmpfs.
const ENV_RUNTIME_DIR: &str = "MATRIX_APPSERVICE_RUNTIME_DIR";
/// The directory earlier versions kept the tokens in. Read once, never written.
const ENV_LEGACY_DIR: &str = "MATRIX_APPSERVICE_LEGACY_DIR";
/// The registration's `sender_localpart`: the account the homeserver creates
/// for it and the one `admin_execute` promotes.
const ENV_SENDER: &str = "MATRIX_APPSERVICE_SENDER";
/// The user namespace regex, rendered by nix.
const ENV_USER_REGEX: &str = "MATRIX_APPSERVICE_USER_REGEX";
/// Role on the store's `cert` auth mount that both verbs log in with.
const ENV_CERT_ROLE: &str = "MATRIX_APPSERVICE_CERT_ROLE";
/// The unit that loads the registration, which [`sync`] restarts when the
/// registration is newer than its last start.
const ENV_RESTART_UNIT: &str = "MATRIX_APPSERVICE_RESTART_UNIT";

/// The registration's `id`.
const ID: &str = "swarm";
/// `REGISTRATION` is what tuwunel loads, in [`ENV_RUNTIME_DIR`]. All three are
/// the names earlier versions used in [`ENV_LEGACY_DIR`].
const AS_TOKEN: &str = "as-token";
const HS_TOKEN: &str = "hs-token";
const REGISTRATION: &str = "swarm.yaml";
/// Random bytes per token.
const TOKEN_BYTES: usize = 32;

/// Read a required variable.
fn required(get: &impl Fn(&str) -> Option<String>, var: &'static str) -> Result<String> {
    get(var)
        .filter(|v| !v.is_empty())
        .with_context(|| format!("{var} is unset or empty"))
}

/// The store, logged in as matrix-ctl.
async fn connect(cert_role: &str) -> Result<Bao> {
    let settings = Settings::from_env().context("reading the store's BAO_* environment")?;
    let store = SecretStore::connect(&settings, cert_role, DEFAULT_CERT_MOUNT)
        .await
        .with_context(|| {
            format!("logging in to the swarm secret store as cert role {cert_role}")
        })?;
    Ok(Bao(store))
}

/// Render the registration from the store's tokens, which are minted first
/// when the store holds none.
///
/// # Errors
/// If a variable is missing, or the store refuses the login, a read or a
/// write, or the runtime dir cannot be written.
pub async fn render() -> Result<()> {
    let get = |k: &str| std::env::var(k).ok();
    let runtime = PathBuf::from(required(&get, ENV_RUNTIME_DIR)?);
    let legacy = PathBuf::from(required(&get, ENV_LEGACY_DIR)?);
    let sender = required(&get, ENV_SENDER)?;
    let regex = required(&get, ENV_USER_REGEX)?;
    let store = connect(&required(&get, ENV_CERT_ROLE)?).await?;
    render_with(&store, &runtime, &legacy, &sender, &regex, mint).await?;
    tracing::info!(path = %runtime.join(REGISTRATION).display(), "rendered the swarm appservice registration");
    Ok(())
}

/// [`render`] against an explicit store and directories.
async fn render_with(
    store: &impl Store,
    runtime: &Path,
    legacy: &Path,
    sender: &str,
    regex: &str,
    mint: impl FnOnce() -> Result<Registration>,
) -> Result<()> {
    let current = resolve(store, legacy, mint).await?;
    write_secret(
        &runtime.join(REGISTRATION),
        &registration_yaml(sender, regex, &current.as_token, &current.hs_token),
    )
}

/// The trimmed token at `path`; `None` when the file is missing or blank.
fn read_token(path: &Path) -> Result<Option<String>> {
    match std::fs::read_to_string(path) {
        Ok(t) if !t.trim().is_empty() => Ok(Some(t.trim().to_owned())),
        Ok(_) => Ok(None),
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
        Err(e) => Err(e).with_context(|| format!("reading {}", path.display())),
    }
}

/// A temp path beside `path`, unique per call and per process, so two writers
/// of the same file never share one. Same directory, so the rename stays
/// atomic.
fn temp_path(path: &Path) -> PathBuf {
    static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
    let mut name = path.file_name().unwrap_or_default().to_os_string();
    name.push(format!(
        ".{}-{}.tmp",
        std::process::id(),
        NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
    ));
    path.with_file_name(name)
}

/// Write `contents` to `path` as a `0600` file, through a rename so a reader
/// never sees half of it. A failure removes the temp file.
fn write_secret(path: &Path, contents: &str) -> Result<()> {
    let tmp = temp_path(path);
    let written = write_temp(&tmp, contents).and_then(|()| {
        std::fs::rename(&tmp, path).with_context(|| format!("renaming onto {}", path.display()))
    });
    if written.is_err() {
        let _ = std::fs::remove_file(&tmp);
    }
    written
}

fn write_temp(tmp: &Path, contents: &str) -> Result<()> {
    let mut f = std::fs::OpenOptions::new()
        .write(true)
        .create_new(true)
        .mode(0o600)
        .open(tmp)
        .with_context(|| format!("creating {}", tmp.display()))?;
    f.write_all(contents.as_bytes())
        .and_then(|()| f.sync_all())
        .with_context(|| format!("writing {}", tmp.display()))
}

/// The registration tuwunel loads.
///
/// `exclusive: false` for the reason `hive-matrix.nix` gives at
/// `appserviceUserRegex`: an exclusive namespace does not widen what this
/// appservice may do, it refuses everyone else.
fn registration_yaml(sender: &str, regex: &str, as_token: &str, hs_token: &str) -> String {
    format!(
        "id: {ID}\n\
         url: null\n\
         sender_localpart: {sender}\n\
         rate_limited: false\n\
         namespaces:\n  \
           users:\n    \
             - exclusive: false\n      \
               regex: '{regex}'\n  \
           aliases: []\n  \
           rooms: []\n\
         as_token: {as_token}\n\
         hs_token: {hs_token}\n"
    )
}

/// What [`render`] and [`sync`] need of the store, so the decisions can be tested without
/// one.
trait Store {
    /// The registration's tokens, or `None` when nothing is stored.
    async fn registration(&self) -> Result<Option<Registration>>;
    /// Store `r` only if nothing is stored yet.
    async fn create_registration(&self, r: &Registration) -> Result<()>;
    /// What `swarm-controller` reads.
    async fn controller_token(&self) -> Result<Option<matrix::Credential>>;
    async fn write_controller_token(&self, as_token: &str) -> Result<()>;
}

/// The real store, logged in as matrix-ctl.
struct Bao(SecretStore);

impl Store for Bao {
    async fn registration(&self) -> Result<Option<Registration>> {
        let path = matrix::swarm_appservice_registration_path()?;
        self.0
            .read_optional(&path)
            .await
            .with_context(|| format!("reading {path}"))
    }

    async fn create_registration(&self, r: &Registration) -> Result<()> {
        let path = matrix::swarm_appservice_registration_path()?;
        self.0
            .create(&path, r)
            .await
            .with_context(|| format!("creating {path}"))
    }

    async fn controller_token(&self) -> Result<Option<matrix::Credential>> {
        let path = matrix::swarm_appservice_token_path()?;
        self.0
            .read_optional(&path)
            .await
            .with_context(|| format!("reading {path}"))
    }

    async fn write_controller_token(&self, as_token: &str) -> Result<()> {
        let path = matrix::swarm_appservice_token_path()?;
        self.0
            .write(
                &path,
                &matrix::Credential {
                    value: as_token.to_owned(),
                    homeserver: None,
                },
            )
            .await
            .with_context(|| format!("writing {path}"))
    }
}

/// Make the runtime dir hold the store's registration, and the controller's
/// token match it.
///
/// # Errors
/// If a variable is missing, the store refuses the login, a read or a write,
/// the runtime dir cannot be written, or the homeserver cannot be restarted.
pub async fn sync() -> Result<()> {
    let get = |k: &str| std::env::var(k).ok();
    let runtime = PathBuf::from(required(&get, ENV_RUNTIME_DIR)?);
    let legacy = PathBuf::from(required(&get, ENV_LEGACY_DIR)?);
    let sender = required(&get, ENV_SENDER)?;
    let regex = required(&get, ENV_USER_REGEX)?;
    let unit = required(&get, ENV_RESTART_UNIT)?;
    let store = connect(&required(&get, ENV_CERT_ROLE)?).await?;
    sync_with(
        &store,
        &runtime,
        &legacy,
        &sender,
        &regex,
        mint,
        &Systemd(unit),
    )
    .await
}

/// [`sync`] against an explicit store, directories and homeserver unit.
///
/// The homeserver is restarted whenever the registration was written after it
/// last started, so a run that failed between the write and the restart is
/// finished by the next.
async fn sync_with(
    store: &impl Store,
    runtime: &Path,
    legacy: &Path,
    sender: &str,
    regex: &str,
    mint: impl FnOnce() -> Result<Registration>,
    unit: &impl Homeserver,
) -> Result<()> {
    let current = resolve(store, legacy, mint).await?;

    let path = runtime.join(REGISTRATION);
    let want = registration_yaml(sender, regex, &current.as_token, &current.hs_token);
    if std::fs::read_to_string(&path).ok().as_deref() != Some(&want) {
        write_secret(&path, &want)?;
        tracing::info!("took the swarm appservice registration from the store");
    }
    restart_if_stale(unit, &path)?;

    if needs_publish(store.controller_token().await?.as_ref(), &current.as_token) {
        store.write_controller_token(&current.as_token).await?;
        tracing::info!("published the swarm appservice token for the controller");
    }
    Ok(())
}

/// The registration the store holds. Where it holds none, the tokens an
/// earlier version kept in `legacy` are stored once (create-only, so nothing
/// rotates), and only a host with neither mints a pair. A store that cannot
/// be read is an error and mints nothing. The files stay where they are,
/// unused.
async fn resolve(
    store: &impl Store,
    legacy: &Path,
    mint: impl FnOnce() -> Result<Registration>,
) -> Result<Registration> {
    match store.registration().await? {
        Some(stored) => Ok(stored),
        None => seed_store(store, read_local(legacy)?.as_ref(), mint).await,
    }
}

/// Create the registration in the empty store from `local`, or from a minted
/// pair when there are no files, and return what the store then holds. A
/// refused create is a lost race or a denial, and the read-back tells which:
/// it finds the winner's pair, or nothing.
async fn seed_store(
    store: &impl Store,
    local: Option<&Registration>,
    mint: impl FnOnce() -> Result<Registration>,
) -> Result<Registration> {
    let seed = match local {
        Some(l) => l.clone(),
        None => mint()?,
    };
    if let Err(e) = store.create_registration(&seed).await {
        tracing::warn!(error = %e, "could not create the swarm appservice registration; reading it back");
    }
    store
        .registration()
        .await?
        .context("the store holds no swarm appservice registration after creating one")
}

/// The tokens in `dir`, when both are there.
fn read_local(dir: &Path) -> Result<Option<Registration>> {
    let as_token = read_token(&dir.join(AS_TOKEN))?;
    let hs_token = read_token(&dir.join(HS_TOKEN))?;
    Ok(as_token
        .zip(hs_token)
        .map(|(as_token, hs_token)| Registration { as_token, hs_token }))
}

fn mint() -> Result<Registration> {
    Ok(Registration {
        as_token: swarm_matrix_client::random_hex(TOKEN_BYTES)?,
        hs_token: swarm_matrix_client::random_hex(TOKEN_BYTES)?,
    })
}

/// Whether the store needs the local token written to it.
fn needs_publish(stored: Option<&matrix::Credential>, local: &str) -> bool {
    stored.is_none_or(|c| c.value.trim() != local)
}

/// What [`sync`] needs of the unit that loads the registration.
trait Homeserver {
    /// When the unit last became active, in microseconds since the epoch;
    /// `None` when it never has.
    fn started_us(&self) -> Result<Option<u64>>;
    /// Restart it onto the registration on disk.
    fn restart(&self) -> Result<()>;
}

/// The real unit, through `systemctl`.
struct Systemd(String);

impl Homeserver for Systemd {
    fn started_us(&self) -> Result<Option<u64>> {
        let out = std::process::Command::new("systemctl")
            .args([
                "show",
                "--timestamp=us+utc",
                "-p",
                "ActiveEnterTimestamp",
                "--value",
            ])
            .arg(&self.0)
            .output()
            .context("running systemctl show")?;
        if !out.status.success() {
            bail!("systemctl show {} exited with {}", self.0, out.status);
        }
        parse_timestamp_us(String::from_utf8_lossy(&out.stdout).trim())
    }

    fn restart(&self) -> Result<()> {
        restart_via(&self.0, systemctl)
    }
}

/// Restart the homeserver when the registration on disk is newer than its last
/// start. Derived from the files and the unit on every run rather than from
/// what this run wrote, so a run that failed after writing is retried by the
/// next. A start time that cannot be read leaves the unit as it is: the next
/// run decides again.
fn restart_if_stale(unit: &impl Homeserver, registration: &Path) -> Result<()> {
    let started = match unit.started_us() {
        Ok(started) => started.unwrap_or(0),
        Err(e) => {
            tracing::warn!(error = %e, "cannot tell when the homeserver last started; leaving it as it is");
            return Ok(());
        }
    };
    let written = std::fs::metadata(registration)
        .and_then(|m| m.modified())
        .with_context(|| format!("reading the time of {}", registration.display()))?
        .duration_since(std::time::UNIX_EPOCH)
        .context("the registration is dated before the epoch")?
        .as_micros();
    if written <= u128::from(started) {
        return Ok(());
    }
    tracing::info!("the registration is newer than the homeserver's start; restarting it");
    unit.restart()
        .context("restarting the homeserver onto the new registration")
}

/// `systemctl systemd-timestamp` output (`--timestamp=us+utc`, for example
/// `Tue 2026-10-27 12:00:00.123456 UTC`) as microseconds since the epoch. Empty
/// is `None`: a unit that never became active has no timestamp.
fn parse_timestamp_us(s: &str) -> Result<Option<u64>> {
    if s.is_empty() {
        return Ok(None);
    }
    let bad = || anyhow::anyhow!("unreadable systemd timestamp {s:?}");
    let token = |sep: char| {
        s.split_whitespace()
            .find(|t| t.contains(sep))
            .ok_or_else(bad)
    };
    let numbers = |text: &str, sep: char| -> Result<Vec<i64>> {
        text.split(sep)
            .map(|n| n.parse().map_err(|_| bad()))
            .collect()
    };
    let time = token(':')?;
    let (clock, fraction) = time.split_once('.').unwrap_or((time, ""));
    let [year, month, day] = numbers(token('-')?, '-')?[..] else {
        return Err(bad());
    };
    let [hour, minute, second] = numbers(clock, ':')?[..] else {
        return Err(bad());
    };
    let micros: i64 = format!("{fraction:0<6}")[..6].parse().map_err(|_| bad())?;
    // Days since 1970-01-01 of a proleptic Gregorian date.
    let year = if month <= 2 { year - 1 } else { year };
    let era = year.div_euclid(400);
    let year_of_era = year - era * 400;
    let day_of_year = (153 * ((month + 9) % 12) + 2) / 5 + day - 1;
    let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
    let days = era * 146_097 + day_of_era - 719_468;
    let seconds = days * 86_400 + hour * 3600 + minute * 60 + second;
    u64::try_from(seconds * 1_000_000 + micros)
        .map(Some)
        .map_err(|_| bad())
}

/// Run `systemctl` with `args`, returning its exit status.
fn systemctl(args: &[&str]) -> Result<std::process::ExitStatus> {
    std::process::Command::new("systemctl")
        .args(args)
        .status()
        .with_context(|| format!("running systemctl {}", args.join(" ")))
}

/// Restart `unit` onto the new registration: a running one is restarted, and a
/// failed one — it may have failed for want of the file — is reset and started.
/// One stopped on purpose stays stopped. `--no-block`, so this run does not
/// wait on the homeserver's start. Every call but the `is-failed` probe must
/// exit zero.
fn restart_via(
    unit: &str,
    run: impl Fn(&[&str]) -> Result<std::process::ExitStatus>,
) -> Result<()> {
    let must = |args: &[&str]| {
        let status = run(args)?;
        if status.success() {
            Ok(())
        } else {
            bail!("systemctl {} exited with {status}", args.join(" "))
        }
    };
    if run(&["is-failed", "--quiet", unit])?.success() {
        must(&["reset-failed", unit])?;
        must(&["start", "--no-block", unit])
    } else {
        must(&["try-restart", "--no-block", unit])
    }
}

#[cfg(test)]
mod tests {
    use std::cell::{Cell, RefCell};

    use super::*;

    const REGEX: &str = "^@[a-z0-9._=/-]+:t\\.local$";

    /// A runtime dir and a legacy dir of their own per call; the legacy one
    /// is not created, as on a host that never had the files. A counter, not
    /// the clock, keeps two tests that start in the same clock tick out of one
    /// directory.
    fn scratch() -> (PathBuf, PathBuf) {
        static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
        let root = std::env::temp_dir().join(format!(
            "swarm-appservice-{}-{}",
            std::process::id(),
            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
        ));
        std::fs::remove_dir_all(&root).ok();
        std::fs::create_dir_all(root.join("run")).expect("temp dir");
        (root.join("run"), root.join("legacy"))
    }

    fn cleanup(runtime: &Path) {
        std::fs::remove_dir_all(runtime.parent().expect("a parent")).ok();
    }

    fn reg(tag: &str) -> Registration {
        Registration {
            as_token: format!("as-{tag}"),
            hs_token: format!("hs-{tag}"),
        }
    }

    /// A store in memory. `down` fails every call, as a sealed or unreachable
    /// store does; `deny_create` refuses a create and stores nothing; `racer`
    /// is what another writer stores between this run's read and its create.
    #[derive(Default)]
    struct Fake {
        registration: RefCell<Option<Registration>>,
        token: RefCell<Option<String>>,
        down: bool,
        deny_create: bool,
        racer: RefCell<Option<Registration>>,
        creates: Cell<u32>,
        token_writes: Cell<u32>,
    }

    impl Fake {
        fn holding(r: Registration) -> Self {
            let f = Self::default();
            *f.registration.borrow_mut() = Some(r);
            f
        }

        fn down() -> Self {
            Self {
                down: true,
                ..Self::default()
            }
        }
    }

    impl Store for Fake {
        async fn registration(&self) -> Result<Option<Registration>> {
            if self.down {
                bail!("the store is down");
            }
            Ok(self.registration.borrow().clone())
        }

        async fn create_registration(&self, r: &Registration) -> Result<()> {
            self.creates.set(self.creates.get() + 1);
            if self.deny_create {
                bail!("permission denied");
            }
            if let Some(winner) = self.racer.borrow_mut().take() {
                *self.registration.borrow_mut() = Some(winner);
                bail!("already created");
            }
            *self.registration.borrow_mut() = Some(r.clone());
            Ok(())
        }

        async fn controller_token(&self) -> Result<Option<matrix::Credential>> {
            Ok(self.token.borrow().clone().map(|value| matrix::Credential {
                value,
                homeserver: None,
            }))
        }

        async fn write_controller_token(&self, as_token: &str) -> Result<()> {
            self.token_writes.set(self.token_writes.get() + 1);
            *self.token.borrow_mut() = Some(as_token.to_owned());
            Ok(())
        }
    }

    /// The three files an earlier version left in the legacy dir for `r`.
    fn legacy_with(dir: &Path, r: &Registration) {
        std::fs::create_dir_all(dir).expect("legacy dir");
        write_secret(&dir.join(AS_TOKEN), &r.as_token).expect("as");
        write_secret(&dir.join(HS_TOKEN), &r.hs_token).expect("hs");
        write_secret(
            &dir.join(REGISTRATION),
            &registration_yaml("swarm", REGEX, &r.as_token, &r.hs_token),
        )
        .expect("yaml");
    }

    /// What is left of the legacy files: the dir's file names, sorted.
    fn legacy_left(dir: &Path) -> Vec<String> {
        let mut names: Vec<String> = std::fs::read_dir(dir)
            .map(|d| {
                d.map(|e| e.expect("entry").file_name().to_string_lossy().into_owned())
                    .collect()
            })
            .unwrap_or_default();
        names.sort();
        names
    }

    /// The registration the runtime dir holds, or empty.
    fn rendered(runtime: &Path) -> String {
        std::fs::read_to_string(runtime.join(REGISTRATION)).unwrap_or_default()
    }

    fn yaml_for(r: &Registration) -> String {
        registration_yaml("swarm", REGEX, &r.as_token, &r.hs_token)
    }

    /// The homeserver unit. `started` is when it last became active; a restart
    /// that succeeds makes it later than every file.
    struct Unit {
        started: Cell<Option<u64>>,
        fail_restart: Cell<bool>,
        restarts: Cell<u32>,
    }

    impl Unit {
        /// Last started at the epoch, before any file in a test.
        fn long_running() -> Self {
            Self::started(Some(1))
        }

        /// Last started after every file in a test.
        fn just_started() -> Self {
            Self::started(Some(u64::MAX))
        }

        fn started(at: Option<u64>) -> Self {
            Self {
                started: Cell::new(at),
                fail_restart: Cell::new(false),
                restarts: Cell::new(0),
            }
        }
    }

    impl Homeserver for Unit {
        fn started_us(&self) -> Result<Option<u64>> {
            Ok(self.started.get())
        }

        fn restart(&self) -> Result<()> {
            self.restarts.set(self.restarts.get() + 1);
            if self.fail_restart.get() {
                bail!("systemctl refused");
            }
            self.started.set(Some(u64::MAX));
            Ok(())
        }
    }

    /// Run one render, returning how often it minted.
    async fn render_run(store: &Fake, runtime: &Path, legacy: &Path) -> (Result<()>, u32) {
        let mints = Cell::new(0);
        let out = render_with(store, runtime, legacy, "swarm", REGEX, || {
            mints.set(mints.get() + 1);
            Ok(reg("minted"))
        })
        .await;
        (out, mints.get())
    }

    /// Run one sync against `unit`, returning how often it minted.
    async fn run(store: &Fake, runtime: &Path, legacy: &Path, unit: &Unit) -> (Result<()>, u32) {
        let mints = Cell::new(0);
        let out = sync_with(
            store,
            runtime,
            legacy,
            "swarm",
            REGEX,
            || {
                mints.set(mints.get() + 1);
                Ok(reg("minted"))
            },
            unit,
        )
        .await;
        (out, mints.get())
    }

    #[tokio::test]
    async fn an_empty_store_and_no_files_mint_a_pair_into_the_store() {
        let (runtime, legacy) = scratch();
        let store = Fake::default();

        let (out, mints) = render_run(&store, &runtime, &legacy).await;

        out.expect("renders");
        assert_eq!(mints, 1);
        assert_eq!(*store.registration.borrow(), Some(reg("minted")));
        assert_eq!(rendered(&runtime), yaml_for(&reg("minted")));
        assert!(!legacy.exists(), "nothing is written to the legacy dir");
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn an_empty_store_takes_the_files_value_once() {
        let (runtime, legacy) = scratch();
        legacy_with(&legacy, &reg("old"));
        let store = Fake::default();

        let (out, mints) = render_run(&store, &runtime, &legacy).await;

        out.expect("renders");
        assert_eq!(mints, 0, "a host that has the value mints nothing");
        assert_eq!(*store.registration.borrow(), Some(reg("old")));
        assert_eq!(rendered(&runtime), yaml_for(&reg("old")));
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_store_that_is_down_mints_nothing_and_renders_nothing() {
        let (runtime, legacy) = scratch();
        legacy_with(&legacy, &reg("disk"));

        let (out, mints) = render_run(&Fake::down(), &runtime, &legacy).await;

        assert!(out.is_err(), "the unit fails and retries");
        assert_eq!(mints, 0);
        assert_eq!(rendered(&runtime), "", "nothing for tuwunel to load");
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_store_that_is_down_on_a_fresh_host_mints_nothing() {
        let (runtime, legacy) = scratch();

        let (out, mints) = render_run(&Fake::down(), &runtime, &legacy).await;

        assert!(out.is_err());
        assert_eq!(
            mints, 0,
            "a pair minted while the store is away is a rotation"
        );
        assert_eq!(rendered(&runtime), "");
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_lost_race_adopts_the_winners_pair() {
        let (runtime, legacy) = scratch();
        legacy_with(&legacy, &reg("disk"));
        let store = Fake::default();
        *store.racer.borrow_mut() = Some(reg("winner"));

        let (out, _) = render_run(&store, &runtime, &legacy).await;

        out.expect("renders");
        assert_eq!(rendered(&runtime), yaml_for(&reg("winner")));
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_refused_create_fails_the_run() {
        let (runtime, legacy) = scratch();
        legacy_with(&legacy, &reg("disk"));
        let store = Fake {
            deny_create: true,
            ..Fake::default()
        };

        let (out, _) = render_run(&store, &runtime, &legacy).await;

        assert!(out.is_err(), "no read-back finds the pair");
        assert_eq!(rendered(&runtime), "");
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_second_render_keeps_the_pair() {
        // "Only once": a re-mint on every boot would hand the controller a
        // token the homeserver no longer loads until sync catches up.
        let (runtime, legacy) = scratch();
        let store = Fake::default();
        render_run(&store, &runtime, &legacy)
            .await
            .0
            .expect("first");
        let first = rendered(&runtime);

        let (out, mints) = render_run(&store, &runtime, &legacy).await;

        out.expect("second");
        assert_eq!(mints, 0);
        assert_eq!(rendered(&runtime), first);
        cleanup(&runtime);
    }

    #[test]
    fn two_writers_of_one_file_use_different_temp_files() {
        let path = Path::new("/run/x/swarm.yaml");
        let (a, b) = (temp_path(path), temp_path(path));
        assert_ne!(a, b);
        assert_eq!(
            a.parent(),
            path.parent(),
            "the rename stays in one directory"
        );
    }

    #[test]
    fn overlapping_writers_all_succeed_and_leave_no_temp_file() {
        let (runtime, _) = scratch();
        let path = runtime.join(REGISTRATION);
        let writers: Vec<_> = (0..8)
            .map(|_| {
                let path = path.clone();
                std::thread::spawn(move || {
                    (0..50)
                        .try_for_each(|_| write_secret(&path, "same\n"))
                        .is_ok()
                })
            })
            .collect();
        assert!(writers.into_iter().all(|w| w.join().expect("joins")));
        assert_eq!(std::fs::read_to_string(&path).expect("written"), "same\n");
        assert_eq!(legacy_left(&runtime), [REGISTRATION]);
        cleanup(&runtime);
    }

    #[test]
    fn a_failed_write_removes_its_temp_file() {
        let (runtime, _) = scratch();
        // A directory where the file goes: the rename fails.
        std::fs::create_dir(runtime.join(REGISTRATION)).expect("dir");
        write_secret(&runtime.join(REGISTRATION), "x").expect_err("cannot replace a directory");
        assert_eq!(legacy_left(&runtime), [REGISTRATION]);
        cleanup(&runtime);
    }

    #[test]
    fn the_registration_is_the_swarms_and_not_the_hives() {
        let y = registration_yaml("swarm", REGEX, "aa", "bb");
        assert!(y.starts_with("id: swarm\n"), "{y}");
        assert!(y.contains("\nsender_localpart: swarm\n"), "{y}");
        assert!(y.contains("\n    - exclusive: false\n"), "{y}");
        assert!(y.contains(&format!("regex: '{REGEX}'")), "{y}");
        assert!(y.contains("\nurl: null\n"), "{y}");
        assert!(y.contains("\nas_token: aa\n"), "{y}");
        assert!(y.contains("\nhs_token: bb\n"), "{y}");
    }

    #[tokio::test]
    async fn the_rendered_registration_is_owner_only() {
        use std::os::unix::fs::PermissionsExt as _;
        let (runtime, legacy) = scratch();
        render_run(&Fake::default(), &runtime, &legacy)
            .await
            .0
            .expect("renders");
        let mode = std::fs::metadata(runtime.join(REGISTRATION))
            .expect("exists")
            .permissions()
            .mode();
        assert_eq!(mode & 0o777, 0o600);
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn the_store_wins_when_the_rendered_copy_differs() {
        let (runtime, legacy) = scratch();
        std::fs::write(runtime.join(REGISTRATION), yaml_for(&reg("old"))).expect("rendered");
        let store = Fake::holding(reg("store"));
        *store.token.borrow_mut() = Some("as-old".to_owned());

        let unit = Unit::long_running();
        let (out, _) = run(&store, &runtime, &legacy, &unit).await;

        out.expect("syncs");
        assert_eq!(rendered(&runtime), yaml_for(&reg("store")));
        assert_eq!(*store.registration.borrow(), Some(reg("store")));
        assert_eq!(store.creates.get(), 0, "a held value is never rewritten");
        assert_eq!(unit.restarts.get(), 1);
        assert_eq!(store.token.borrow().as_deref(), Some("as-store"));
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_sync_with_nothing_rendered_takes_the_stores_pair() {
        let (runtime, legacy) = scratch();
        let store = Fake::holding(reg("store"));

        let unit = Unit::long_running();
        let (out, mints) = run(&store, &runtime, &legacy, &unit).await;

        out.expect("syncs");
        assert_eq!(rendered(&runtime), yaml_for(&reg("store")));
        assert_eq!((unit.restarts.get(), mints), (1, 0));
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_store_that_is_down_leaves_the_rendered_copy_as_it_is() {
        let (runtime, legacy) = scratch();
        std::fs::write(runtime.join(REGISTRATION), yaml_for(&reg("disk"))).expect("rendered");

        let unit = Unit::just_started();
        let (out, mints) = run(&Fake::down(), &runtime, &legacy, &unit).await;

        assert!(out.is_err(), "the unit reports the outage");
        assert_eq!(rendered(&runtime), yaml_for(&reg("disk")));
        assert_eq!((unit.restarts.get(), mints), (0, 0));
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_second_sync_restarts_nothing() {
        let (runtime, legacy) = scratch();
        let store = Fake::holding(reg("store"));

        let unit = Unit::long_running();

        run(&store, &runtime, &legacy, &unit)
            .await
            .0
            .expect("first");
        let first = unit.restarts.get();
        let (out, _) = run(&store, &runtime, &legacy, &unit).await;
        let second = unit.restarts.get() - first;

        out.expect("syncs");
        assert_eq!((first, second), (1, 0));
        assert_eq!(
            store.token_writes.get(),
            1,
            "the controller's copy is written once"
        );
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_restart_that_failed_is_retried_by_the_next_run() {
        // The file is written and the restart is refused. The next run finds
        // nothing to write, and still restarts: the decision comes from the
        // file's age against the unit's start, not from this run's writes.
        let (runtime, legacy) = scratch();
        let store = Fake::holding(reg("store"));
        let unit = Unit::long_running();
        unit.fail_restart.set(true);

        let (first, _) = run(&store, &runtime, &legacy, &unit).await;
        assert!(first.is_err(), "the refused restart fails the run");
        let written = rendered(&runtime);

        unit.fail_restart.set(false);
        let (second, _) = run(&store, &runtime, &legacy, &unit).await;

        second.expect("the retry succeeds");
        assert_eq!(rendered(&runtime), written, "nothing was left to write");
        assert_eq!(unit.restarts.get(), 2);
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_unit_that_started_after_the_file_is_left_alone() {
        // The control for the retry above: the same file, a unit that started
        // after it, and no restart.
        let (runtime, legacy) = scratch();
        std::fs::write(runtime.join(REGISTRATION), yaml_for(&reg("store"))).expect("rendered");
        let store = Fake::holding(reg("store"));
        let unit = Unit::just_started();

        run(&store, &runtime, &legacy, &unit)
            .await
            .0
            .expect("syncs");

        assert_eq!(unit.restarts.get(), 0);
        cleanup(&runtime);
    }

    #[tokio::test]
    async fn a_start_time_that_cannot_be_read_restarts_nothing() {
        struct Unreadable(Cell<u32>);
        impl Homeserver for Unreadable {
            fn started_us(&self) -> Result<Option<u64>> {
                bail!("no such unit")
            }
            fn restart(&self) -> Result<()> {
                self.0.set(self.0.get() + 1);
                Ok(())
            }
        }
        let (runtime, legacy) = scratch();
        let store = Fake::holding(reg("store"));
        let unit = Unreadable(Cell::new(0));

        let out = sync_with(
            &store,
            &runtime,
            &legacy,
            "swarm",
            REGEX,
            || Ok(reg("minted")),
            &unit,
        )
        .await;

        out.expect("the run goes on");
        assert_eq!(unit.0.get(), 0);
        assert_eq!(store.token.borrow().as_deref(), Some("as-store"));
        cleanup(&runtime);
    }

    /// `restart_via` against a `systemctl` that exits with the given codes for the
    /// verbs named (zero for the rest), returning the verbs it was called with.
    fn restart_script(exits: &[(&'static str, i32)]) -> (Result<()>, Vec<String>) {
        use std::os::unix::process::ExitStatusExt as _;
        let calls = RefCell::new(Vec::new());
        let out = restart_via("tuwunel.service", |args| {
            calls.borrow_mut().push(args[0].to_owned());
            let code = exits.iter().find(|(v, _)| *v == args[0]).map_or(0, |e| e.1);
            Ok(std::process::ExitStatus::from_raw(code << 8))
        });
        (out, calls.into_inner())
    }

    #[test]
    fn a_running_unit_is_restarted_and_a_refusal_fails_the_run() {
        let (out, calls) = restart_script(&[("is-failed", 1)]);
        out.expect("restarts");
        assert_eq!(calls, ["is-failed", "try-restart"]);

        let (out, _) = restart_script(&[("is-failed", 1), ("try-restart", 1)]);
        assert!(out.is_err());
    }

    #[test]
    fn a_failed_unit_is_reset_and_started_and_a_refusal_of_either_fails_the_run() {
        let (out, calls) = restart_script(&[]);
        out.expect("resets and starts");
        assert_eq!(calls, ["is-failed", "reset-failed", "start"]);

        let (out, calls) = restart_script(&[("reset-failed", 1)]);
        assert!(out.is_err(), "a refused reset fails the run");
        assert_eq!(calls, ["is-failed", "reset-failed"], "and nothing starts");

        let (out, _) = restart_script(&[("start", 1)]);
        assert!(out.is_err(), "a refused start fails the run");
    }

    #[test]
    fn systemd_timestamps_read_as_microseconds_since_the_epoch() {
        let at = |s| parse_timestamp_us(s).expect("parses");
        assert_eq!(at("Thu 1970-01-01 00:00:01.000002 UTC"), Some(1_000_002));
        assert_eq!(
            at("Wed 2000-03-01 00:00:00.5 UTC"),
            Some(951_868_800_500_000)
        );
        assert_eq!(
            at("Tue 2026-10-27 12:00:00 UTC"),
            Some(1_793_102_400_000_000)
        );
        assert_eq!(at(""), None, "a unit that never started has none");
        parse_timestamp_us("n/a garbage").expect_err("not a timestamp");
    }

    #[test]
    fn the_controllers_copy_is_written_only_when_it_differs() {
        let same = matrix::Credential {
            value: "aa".to_owned(),
            homeserver: None,
        };
        assert!(!needs_publish(Some(&same), "aa"));
        assert!(needs_publish(None, "aa"));
        let other = matrix::Credential {
            value: "bb".to_owned(),
            homeserver: None,
        };
        assert!(needs_publish(Some(&other), "aa"));
    }

    #[test]
    fn every_variable_is_scoped_to_the_verb() {
        for var in [
            ENV_RUNTIME_DIR,
            ENV_LEGACY_DIR,
            ENV_SENDER,
            ENV_USER_REGEX,
            ENV_CERT_ROLE,
            ENV_RESTART_UNIT,
        ] {
            assert!(var.starts_with("MATRIX_APPSERVICE_"), "{var}");
        }
    }
}