hyperhive/swarm-secret-client/src/client.rs

//! A logged-in handle on the store, built from this deployment's environment.

use std::time::Duration;

use rustify_derive::Endpoint;
use serde::{Serialize, de::DeserializeOwned};
use vaultrs::{
    api::pki::{requests::GenerateCertificateRequest, responses::GenerateCertificateResponse},
    client::{Client, VaultClient, VaultClientSettings, VaultClientSettingsBuilder},
};

use crate::{Error, mtls, path::MOUNT};

/// The store's address.
pub const ENV_ADDR: &str = "BAO_ADDR";
/// PEM client certificate presented to the store's listener.
pub const ENV_CLIENT_CERT: &str = "BAO_CLIENT_CERT";
/// PEM private key for [`ENV_CLIENT_CERT`].
pub const ENV_CLIENT_KEY: &str = "BAO_CLIENT_KEY";
/// CA bundle the store's own certificate is verified against. Optional:
/// absent means the system trust store, which is what a deployment with a
/// real CA wants and what a self-signed one must not be left with.
pub const ENV_CACERT: &str = "BAO_CACERT";

/// The auth mount a cert login goes through, unless a caller says otherwise.
pub const DEFAULT_CERT_MOUNT: &str = "cert";

/// Upper bound on one request to the store, login included.
///
/// vaultrs sets no timeout by default, so a store that accepts the connection
/// and never answers would hold every [`SecretStore`] call open forever; with
/// this bound the call fails with [`Error::Vault`] instead.
const VAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);

/// Where the store is and which identity we present to it.
///
/// Separate from the connect so the environment can be checked without one:
/// every arm below is a misconfiguration an operator has to read an error
/// about, and none of them needs a reachable store to happen.
#[derive(Debug, PartialEq, Eq)]
pub struct Settings {
    address: String,
    cert_path: String,
    key_path: String,
    ca_path: Option<String>,
}

impl Settings {
    /// The vaultrs settings for this store, presenting `identity`.
    fn client_settings(
        &self,
        identity: Option<reqwest::Identity>,
    ) -> Result<VaultClientSettings, Error> {
        let mut builder = VaultClientSettingsBuilder::default();
        builder
            .address(self.address.clone())
            .identity(identity)
            .timeout(Some(VAULT_REQUEST_TIMEOUT));
        if let Some(ca) = &self.ca_path {
            builder.ca_certs(vec![ca.clone()]);
        }
        builder.build().map_err(|e| Error::Settings(e.to_string()))
    }

    /// Read the `BAO_*` variables from the process environment.
    ///
    /// # Errors
    /// [`Error::MissingEnv`] naming the first variable that is unset or empty.
    pub fn from_env() -> Result<Self, Error> {
        Self::from_lookup(|k| std::env::var(k).ok())
    }

    /// [`Settings::from_env`] against an arbitrary lookup.
    ///
    /// [`vaultrs`]'s own defaults are deliberately not used: it looks for
    /// `VAULT_ADDR` / `VAULT_CLIENT_CERT` / `VAULT_CLIENT_KEY`, and every unit
    /// in this tree sets the `BAO_` spellings. Falling through to those
    /// defaults yields a client with **no identity**, which fails at the TLS
    /// handshake rather than anywhere that names the cause.
    ///
    /// # Errors
    /// [`Error::MissingEnv`] naming the first variable that is unset or empty.
    pub fn from_lookup(get: impl Fn(&str) -> Option<String>) -> Result<Self, Error> {
        let required = |var: &'static str| -> Result<String, Error> {
            get(var)
                .filter(|v| !v.is_empty())
                .ok_or(Error::MissingEnv(var))
        };
        Ok(Self {
            address: required(ENV_ADDR)?,
            cert_path: required(ENV_CLIENT_CERT)?,
            key_path: required(ENV_CLIENT_KEY)?,
            ca_path: get(ENV_CACERT).filter(|v| !v.is_empty()),
        })
    }
}

/// A client that has already exchanged its certificate for a token.
pub struct SecretStore {
    inner: VaultClient,
}

fn read_file(var: &'static str, path: &str) -> Result<Vec<u8>, Error> {
    std::fs::read(path).map_err(|source| Error::Identity {
        var,
        path: path.to_owned(),
        source,
    })
}

impl SecretStore {
    /// Connect using the `BAO_*` environment and log in with the certificate
    /// auth method, returning a handle that already holds a token.
    ///
    /// `cert_role` is the role configured on the store's `cert` mount; it
    /// selects which policy the returned token carries.
    ///
    /// # Errors
    /// As [`SecretStore::connect`], plus [`Error::MissingEnv`].
    pub async fn from_env(cert_role: &str) -> Result<Self, Error> {
        Self::connect(&Settings::from_env()?, cert_role, DEFAULT_CERT_MOUNT).await
    }

    /// Present `settings`' identity to the store and log in at `cert_mount`.
    ///
    /// # Errors
    /// [`Error::Identity`] when a named file cannot be read, [`Error::Tls`]
    /// when the cert/key pair does not form a usable identity,
    /// [`Error::Settings`] when the address will not parse, and
    /// [`Error::Vault`] when the store refuses the login — which is what an
    /// unconfigured `cert` mount looks like from here.
    pub async fn connect(
        settings: &Settings,
        cert_role: &str,
        cert_mount: &str,
    ) -> Result<Self, Error> {
        // One PEM blob holding both, which is the shape `Identity::from_pem`
        // wants; the two stay separate on disk because the key is the half
        // that gets `0600`.
        let mut pem = read_file(ENV_CLIENT_CERT, &settings.cert_path)?;
        pem.push(b'\n');
        pem.extend_from_slice(&read_file(ENV_CLIENT_KEY, &settings.key_path)?);
        Self::connect_with_identity(settings, &pem, cert_role, cert_mount).await
    }

    /// Log in at the store `settings` names, presenting an identity the caller
    /// is already holding rather than one named by the environment.
    ///
    /// `identity_pem` is the certificate and its private key concatenated into
    /// one PEM blob — the shape [`reqwest::Identity::from_pem`] takes, and the
    /// shape [`SecretStore::connect`] builds out of the two files
    /// [`ENV_CLIENT_CERT`] and [`ENV_CLIENT_KEY`] name. Only the address and
    /// the CA are read out of `settings` here; its two identity paths are not
    /// touched.
    ///
    /// This is what a **minter** needs. A process that has just issued a leaf
    /// holds the bytes, and the safest place for a freshly minted private key
    /// is the memory it was generated in: writing it to a file purely so that
    /// a `Settings` could name it would put a key on disk for no other reason
    /// than to log in with it once.
    ///
    /// # Errors
    /// [`Error::Tls`] when `identity_pem` is not a usable certificate/key
    /// pair, [`Error::Settings`] when the address will not parse, and
    /// [`Error::Vault`] when the store refuses the login — which is what a
    /// certificate no cert-auth role accepts looks like from here.
    pub async fn connect_with_identity(
        settings: &Settings,
        identity_pem: &[u8],
        cert_role: &str,
        cert_mount: &str,
    ) -> Result<Self, Error> {
        let identity = reqwest::Identity::from_pem(identity_pem).map_err(Error::Tls)?;

        let built = settings.client_settings(Some(identity))?;

        let mut inner = VaultClient::new(built)?;
        let auth = vaultrs::auth::cert::login(&inner, cert_mount, cert_role).await?;
        inner.set_token(&auth.client_token);
        Ok(Self { inner })
    }

    /// Read the object stored at `path`, decoded as `T`.
    ///
    /// The shape belongs to whoever owns the path — see [`crate::matrix`] for
    /// the one kind of secret that exists — so this moves bytes and knows no
    /// fields.
    ///
    /// # Errors
    /// [`Error::Vault`] when the path does not exist, the token's policy does
    /// not cover it, or the stored object does not decode as `T`.
    pub async fn read<T: DeserializeOwned>(&self, path: &str) -> Result<T, Error> {
        Ok(vaultrs::kv2::read(&self.inner, MOUNT, path).await?)
    }

    /// Read the object stored at `path`, or `None` when nothing is stored
    /// there.
    ///
    /// The verb for a caller whose write must be a no-op if the path is
    /// already populated. [`read`][Self::read] cannot serve that: it collapses
    /// *absent*, *denied* and *undecodable* into one `Err`, so a caller
    /// treating every failure as absence would overwrite a live credential
    /// whenever the store was merely unreachable.
    ///
    /// **Only a 404 is absence.** A denial is a 403 and stays an error — a
    /// principal whose grant does not cover the path must not conclude the
    /// path is empty, which is how a write ends up clobbering something the
    /// caller was never allowed to see.
    ///
    /// # Errors
    /// [`Error::Vault`] for anything that is not a 404: a denial, an
    /// unreachable store, or an object that does not decode as `T`.
    pub async fn read_optional<T: DeserializeOwned>(&self, path: &str) -> Result<Option<T>, Error> {
        match vaultrs::kv2::read(&self.inner, MOUNT, path).await {
            Ok(value) => Ok(Some(value)),
            Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(None),
            Err(e) => Err(e.into()),
        }
    }

    /// The keys stored directly under `path`, or none when nothing is.
    ///
    /// A key ending in `/` is a directory below `path` rather than an object;
    /// keys come back as the store spells them, so a caller wanting objects
    /// filters those out.
    ///
    /// Addresses `secret/metadata/<path>`, which the store ACLs separately
    /// from the `secret/data/<path>` that [`read`][Self::read] uses: a token
    /// that may read every object under a path is refused here until its
    /// policy grants `list` on the metadata path too.
    ///
    /// **Only a 404 is "none"**, as in [`read_optional`][Self::read_optional]:
    /// the store answers a `LIST` on an empty directory with a 404, and a
    /// denial stays an error.
    ///
    /// # Errors
    /// [`Error::Vault`] for anything that is not a 404: a denial, or an
    /// unreachable store.
    pub async fn list(&self, path: &str) -> Result<Vec<String>, Error> {
        match vaultrs::kv2::list(&self.inner, MOUNT, path).await {
            Ok(keys) => Ok(keys),
            Err(e) if is_absent(&e) => Ok(Vec::new()),
            Err(e) => Err(e.into()),
        }
    }

    /// Write `value` at `path`, creating a new version.
    ///
    /// # Errors
    /// [`Error::Vault`] when the token's policy does not cover the path.
    pub async fn write<T: Serialize + Sync>(&self, path: &str, value: &T) -> Result<(), Error> {
        vaultrs::kv2::set(&self.inner, MOUNT, path, value).await?;
        Ok(())
    }

    /// Write `value` at `path` only if nothing is stored there yet.
    ///
    /// KV v2's check-and-set with `cas=0`, so of two writers racing on an
    /// empty path exactly one lands. The other gets an error, and a caller
    /// that only needs *a* value there reads the winner's back.
    ///
    /// # Errors
    /// [`Error::Vault`] when something is already stored at `path`, or the
    /// token's policy does not cover it. Under a grant of `create` without
    /// `update` the first of those is a 403, not a check-and-set mismatch:
    /// the store checks the capability for an existing path before it gets
    /// to `cas`.
    pub async fn create<T: Serialize + Sync>(&self, path: &str, value: &T) -> Result<(), Error> {
        vaultrs::kv2::set_with_options(&self.inner, MOUNT, path, value, create_only()).await?;
        Ok(())
    }

    /// Delete `path` outright: every version of it, and the metadata that
    /// lists them.
    ///
    /// The undo of [`write`][Self::write], and deliberately **not** KV v2's
    /// plain delete. That one soft-deletes the newest version only, leaving
    /// every earlier one readable at `?version=N` and the newest one
    /// recoverable by `undelete` — so a caller revoking a bearer secret that
    /// [`write`][Self::write] has ever replaced would leave the old value
    /// sitting in version history, still usable. `metadata` is the one verb
    /// that removes the lot.
    ///
    /// Addresses `secret/metadata/<path>`, which the store ACLs separately
    /// from the `secret/data/<path>` the other methods here use: a token
    /// granted write on the data path is refused at this one until its policy
    /// names the metadata path too.
    ///
    /// **Idempotent.** Deleting a path that holds nothing succeeds, because
    /// the caller is tearing something down and a teardown that runs twice —
    /// or on a principal that never got a secret — must not fail the second
    /// time. Only a 404 is absence, exactly as in
    /// [`read_optional`][Self::read_optional]: a 403 stays an error, or a
    /// principal whose grant does not cover the path would report a
    /// revocation it never performed.
    ///
    /// # Errors
    /// [`Error::Vault`] for anything that is not a 404 — a denial, or a store
    /// that could not be reached.
    pub async fn delete_all_versions(&self, path: &str) -> Result<(), Error> {
        match vaultrs::kv2::delete_metadata(&self.inner, MOUNT, path).await {
            Ok(()) => Ok(()),
            Err(e) if is_absent(&e) => Ok(()),
            Err(e) => Err(e.into()),
        }
    }

    /// Replace the ACL policy named `name` with `policy`.
    ///
    /// A whole-document write, not a merge: the store has no other verb, and
    /// the caller renders the document from the current agent set anyway, so
    /// a stanza that is gone from the render is meant to be gone from the
    /// grant. Render the text with [`crate::policy`] rather than by hand.
    ///
    /// # Errors
    /// [`Error::Vault`] when the token's own policy does not cover
    /// `sys/policies/acl/<name>` — which is what a controller scoped to the
    /// `hive-*` namespace gets for any other name.
    pub async fn write_policy(&self, name: &str, policy: &str) -> Result<(), Error> {
        let request = WriteAclPolicy {
            name: name.to_owned(),
            policy: policy.to_owned(),
        };
        vaultrs::api::exec_with_empty(&self.inner, request).await?;
        Ok(())
    }

    /// Create or replace the cert-auth role `name`, so that a certificate
    /// issued by `certificate` and carrying `common_name` logs in with
    /// `policy`.
    ///
    /// `certificate` is the authority by value, not a path: the store keeps
    /// its own copy inside the role and never reads a file of ours.
    ///
    /// # Errors
    /// [`Error::Vault`] when the token's policy does not cover
    /// `auth/<mount>/certs/<name>`, or the store rejects the authority.
    pub async fn write_cert_role(
        &self,
        mount: &str,
        name: &str,
        certificate: &str,
        common_name: &str,
        policy: &str,
    ) -> Result<(), Error> {
        let mut opts =
            vaultrs::api::auth::cert::requests::CreateCaCertificateRoleRequest::builder();
        opts.allowed_common_names(vec![common_name.to_owned()])
            .token_policies(vec![policy.to_owned()]);
        vaultrs::auth::cert::ca_cert_role::set(
            &self.inner,
            mount,
            name,
            certificate,
            Some(&mut opts),
        )
        .await?;
        Ok(())
    }

    /// Have the PKI role `role` on `mount` issue a client certificate for
    /// `common_name`, returning it with its key and the issuing CA.
    ///
    /// The store generates the key: it exists in the store's answer and in the
    /// returned value, nowhere else. What the certificate may carry (names,
    /// usages, lifetime) is the role's to decide, so nothing but the name is
    /// asked for here.
    ///
    /// # Errors
    /// [`Error::Vault`] when the token's policy does not cover
    /// `<mount>/issue/<role>` or the role refuses `common_name`, and
    /// [`Error::IncompleteIssue`] when the answer lacks any of the three.
    pub async fn issue_client_certificate(
        &self,
        mount: &str,
        role: &str,
        common_name: &str,
    ) -> Result<mtls::Credential, Error> {
        let issued =
            vaultrs::api::exec_with_result(&self.inner, issue_request(mount, role, common_name))
                .await?;
        credential_from_issue(issued)
    }

    /// The names of every cert-auth role under `mount`, or none when the
    /// mount has no roles at all.
    ///
    /// **Only a 404 is "none".** The store answers a `LIST` on an empty
    /// directory with a 404; a denial is a 403 and stays an error, so a caller
    /// iterating the result never mistakes "not allowed to look" for "nothing
    /// there".
    ///
    /// # Errors
    /// [`Error::Vault`] for anything that is not a 404: a denial, or an
    /// unreachable store.
    pub async fn list_cert_roles(&self, mount: &str) -> Result<Vec<String>, Error> {
        match vaultrs::auth::cert::ca_cert_role::list(&self.inner, mount).await {
            Ok(listed) => Ok(listed.keys),
            Err(vaultrs::error::ClientError::APIError { code: 404, .. }) => Ok(Vec::new()),
            Err(e) => Err(e.into()),
        }
    }
}

/// Whether a failed store call means the path holds nothing — the one status
/// [`SecretStore::delete_all_versions`] is allowed to treat as success.
///
/// The same rule [`SecretStore::read_optional`] documents, named here because
/// the delete side has to be able to state it in a test: a 403 is a principal
/// whose grant misses the path, and swallowing that would let a caller report
/// a revocation the store refused to perform.
fn is_absent(e: &vaultrs::error::ClientError) -> bool {
    matches!(e, vaultrs::error::ClientError::APIError { code: 404, .. })
}

/// Write an ACL policy, spelled out here because [`vaultrs`] does not have it:
/// its `sys::policy` module targets `sys/policy/<name>`, the deprecated alias,
/// and the store ACLs that path separately from `sys/policies/acl/<name>`. A
/// token granted the latter — which is what every grant in this tree names —
/// is refused at the former with a 403.
#[derive(Endpoint, Serialize)]
#[endpoint(path = "sys/policies/acl/{self.name}", method = "PUT")]
struct WriteAclPolicy {
    name: String,
    /// Marked as the body so `name` stays in the path alone: an untagged field
    /// would be serialised into the request too.
    #[endpoint(body)]
    policy: String,
}

/// The options [`SecretStore::create`] writes with: version 0, which only a
/// path holding nothing is at.
fn create_only() -> vaultrs::api::kv2::requests::SetSecretRequestOptions {
    vaultrs::api::kv2::requests::SetSecretRequestOptions { cas: 0 }
}

/// The request [`SecretStore::issue_client_certificate`] sends.
///
/// The name stays out of the SANs so the certificate carries exactly one name,
/// the one the cert-auth role matches. PKCS#8 because that is the key shape
/// every reader of the published identity already parses.
fn issue_request(mount: &str, role: &str, common_name: &str) -> GenerateCertificateRequest {
    GenerateCertificateRequest {
        mount: mount.to_owned(),
        role: role.to_owned(),
        common_name: Some(common_name.to_owned()),
        exclude_cn_from_sans: Some(true),
        private_key_format: Some("pkcs8".to_owned()),
        ..GenerateCertificateRequest::default()
    }
}

/// The store's answer, moved straight into the redacting
/// [`mtls::Credential`]. `GenerateCertificateResponse` derives `Debug` and
/// holds the private key, so it is consumed here and never formatted.
///
/// `issuing_ca` rather than `ca_chain` because it is the one certificate a
/// cert-auth role pins: the authority that signed this leaf.
fn credential_from_issue(issued: GenerateCertificateResponse) -> Result<mtls::Credential, Error> {
    for (field, value) in [
        ("certificate", &issued.certificate),
        ("private_key", &issued.private_key),
        ("issuing_ca", &issued.issuing_ca),
    ] {
        if value.trim().is_empty() {
            return Err(Error::IncompleteIssue(field));
        }
    }
    Ok(mtls::Credential {
        cert: issued.certificate,
        key: issued.private_key,
        ca: issued.issuing_ca,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A lookup standing in for a fully-configured unit's environment.
    fn full(k: &str) -> Option<String> {
        match k {
            ENV_ADDR => Some("https://store.invalid:8200".to_owned()),
            ENV_CLIENT_CERT => Some("/run/c.pem".to_owned()),
            ENV_CLIENT_KEY => Some("/run/k.pem".to_owned()),
            _ => None,
        }
    }

    #[test]
    fn the_client_is_built_with_a_request_timeout() {
        let s = Settings::from_lookup(full).expect("every required variable is set");
        let built = s.client_settings(None).expect("settings build");
        assert_eq!(built.timeout, Some(VAULT_REQUEST_TIMEOUT));
    }

    #[test]
    fn a_complete_environment_is_accepted() {
        // The control: without this, every assertion below could be passing
        // because `from_lookup` rejects everything.
        let s = Settings::from_lookup(full).expect("every required variable is set");
        assert_eq!(s.address, "https://store.invalid:8200");
        assert_eq!(s.ca_path, None, "an absent CA is the system trust store");
    }

    #[test]
    fn each_required_variable_is_named_when_it_is_the_missing_one() {
        for var in [ENV_ADDR, ENV_CLIENT_CERT, ENV_CLIENT_KEY] {
            let e = Settings::from_lookup(|k| if k == var { None } else { full(k) })
                .expect_err("one required variable is absent");
            assert!(
                matches!(e, Error::MissingEnv(v) if v == var),
                "dropping {var} should name {var}, got {e:?}"
            );
        }
    }

    #[test]
    fn an_empty_variable_is_as_absent_as_an_unset_one() {
        // systemd writes `Environment=BAO_CACERT=` for an unset nix option, so
        // empty is the shape this actually arrives in.
        let e = Settings::from_lookup(|k| {
            if k == ENV_ADDR {
                Some(String::new())
            } else {
                full(k)
            }
        })
        .expect_err("an empty address is not an address");
        assert!(matches!(e, Error::MissingEnv(ENV_ADDR)), "got {e:?}");

        let s = Settings::from_lookup(|k| {
            if k == ENV_CACERT {
                Some(String::new())
            } else {
                full(k)
            }
        })
        .expect("an empty CA is optional, not fatal");
        assert_eq!(s.ca_path, None);
    }

    #[test]
    fn an_unreadable_cert_names_the_file_and_the_variable() {
        let e = read_file(ENV_CLIENT_CERT, "/nonexistent/cert.pem")
            .expect_err("the file does not exist");
        match e {
            Error::Identity { var, ref path, .. } => {
                assert_eq!(var, ENV_CLIENT_CERT);
                assert_eq!(path, "/nonexistent/cert.pem");
            }
            other => panic!("wanted an Identity error, got {other:?}"),
        }
    }

    /// The defect this pins: the store gates `sys/policy/<name>` and
    /// `sys/policies/acl/<name>` separately, so a client on the first is
    /// refused by a grant naming the second — and nothing else in the tree
    /// says which one is addressed.
    #[test]
    fn a_policy_write_addresses_the_modern_acl_path() {
        use rustify::endpoint::Endpoint as _;

        let request = WriteAclPolicy {
            name: "hive-pr1ma".to_owned(),
            policy: crate::policy::render("pr1ma").expect("a plain name is legal"),
        };
        assert_eq!(request.path(), "sys/policies/acl/hive-pr1ma");
    }

    #[test]
    fn the_request_body_carries_the_policy_and_not_the_name() {
        use rustify::endpoint::Endpoint as _;

        let request = WriteAclPolicy {
            name: "hive-pr1ma".to_owned(),
            policy: crate::policy::render("pr1ma").expect("a plain name is legal"),
        };
        let body = request
            .body()
            .expect("the body serialises")
            .expect("a policy write sends one");
        let sent: serde_json::Value =
            serde_json::from_slice(&body).expect("the body is the JSON the store parses");
        assert_eq!(
            sent["policy"],
            crate::policy::render("pr1ma").expect("legal")
        );
        assert!(
            sent.get("name").is_none(),
            "`name` addresses the policy in the path; sending it too would make \
             the store's copy of the document disagree with its own name"
        );
    }

    /// The controller's grant names exactly this path with `update`, which is
    /// what a POST needs; any other path or verb is a 403.
    #[test]
    fn an_issue_request_posts_to_the_roles_issue_path() {
        use rustify::endpoint::Endpoint as _;

        let request = issue_request("pki-agents", "swarm-agent", "hive-agent-atlas");
        assert_eq!(request.path(), "pki-agents/issue/swarm-agent");
        assert!(
            matches!(request.method(), rustify::enums::RequestMethod::POST),
            "{:?}",
            request.method()
        );
    }

    #[test]
    fn an_issue_request_asks_for_the_name_and_nothing_the_role_decides() {
        use rustify::endpoint::Endpoint as _;

        let body = issue_request("pki-agents", "swarm-agent", "hive-agent-atlas")
            .body()
            .expect("the body serialises")
            .expect("an issue request sends one");
        let sent: serde_json::Value = serde_json::from_slice(&body).expect("JSON");
        assert_eq!(sent["common_name"], "hive-agent-atlas");
        assert_eq!(sent["exclude_cn_from_sans"], true);
        assert_eq!(sent["private_key_format"], "pkcs8");
        for decided_by_the_role in ["ttl", "alt_names", "ip_sans", "uri_sans", "other_sans"] {
            assert!(
                sent.get(decided_by_the_role)
                    .is_none_or(serde_json::Value::is_null),
                "{decided_by_the_role} is the role's to set: {sent}"
            );
        }
    }

    fn issued() -> GenerateCertificateResponse {
        GenerateCertificateResponse {
            ca_chain: None,
            certificate: "LEAF".to_owned(),
            expiration: None,
            issuing_ca: "AGENT-CA".to_owned(),
            private_key: "SECRET-KEY-BYTES".to_owned(),
            private_key_type: "ec".to_owned(),
            serial_number: "01".to_owned(),
        }
    }

    #[test]
    fn a_complete_answer_becomes_the_published_credential() {
        let credential = credential_from_issue(issued()).expect("every field is present");
        assert_eq!(credential.cert, "LEAF");
        assert_eq!(credential.key, "SECRET-KEY-BYTES");
        assert_eq!(credential.ca, "AGENT-CA", "the role pins the issuing CA");
        assert!(
            !format!("{credential:?}").contains("SECRET-KEY-BYTES"),
            "the key must not reach a formatted value"
        );
    }

    #[test]
    fn an_answer_missing_any_part_of_the_identity_is_refused_by_name() {
        type Blank = fn(&mut GenerateCertificateResponse);
        let cases: [(&str, Blank); 3] = [
            ("certificate", |r| r.certificate.clear()),
            ("private_key", |r| r.private_key = " \n".to_owned()),
            ("issuing_ca", |r| r.issuing_ca.clear()),
        ];
        for (field, blank) in cases {
            let mut answer = issued();
            blank(&mut answer);
            let e = credential_from_issue(answer).expect_err("an incomplete identity");
            assert!(
                matches!(e, Error::IncompleteIssue(f) if f == field),
                "blanking {field} gave {e:?}"
            );
            assert!(!e.to_string().contains("SECRET-KEY-BYTES"), "{e}");
        }
    }

    /// A create is a write with `cas=0` in the body: without it the store
    /// takes the write as an overwrite of whatever is there, and the two
    /// racing writers [`SecretStore::create`] exists for would both land.
    #[test]
    fn a_create_writes_the_data_path_only_while_it_holds_nothing() {
        use rustify::endpoint::Endpoint as _;

        let request = vaultrs::api::kv2::requests::SetSecretRequest::builder()
            .mount(MOUNT)
            .path("swarm/public/x")
            .data(serde_json::json!({ "value": "v" }))
            .options(create_only())
            .build()
            .expect("the request builds");
        assert_eq!(request.path(), "secret/data/swarm/public/x");
        let body = request
            .body()
            .expect("the body serialises")
            .expect("a write sends one");
        let sent: serde_json::Value = serde_json::from_slice(&body).expect("JSON");
        assert_eq!(sent["options"]["cas"], 0);
        assert_eq!(sent["data"]["value"], "v");
    }

    /// What makes [`SecretStore::delete_all_versions`] safe to run twice: the
    /// second run finds nothing and must still succeed. A teardown that
    /// hard-errors on an already-revoked credential is a worse fault than the
    /// credential outliving its holder.
    #[test]
    fn a_path_that_holds_nothing_is_not_a_failed_delete() {
        assert!(is_absent(&vaultrs::error::ClientError::APIError {
            code: 404,
            errors: vec![],
        }));
    }

    /// The other half, and the one that matters for a revocation: a token
    /// whose policy misses the path is refused with a 403, and reading that
    /// as "nothing was there" would log a revocation the store never made.
    #[test]
    fn a_denial_is_not_absence() {
        for code in [400, 403, 500, 503] {
            assert!(
                !is_absent(&vaultrs::error::ClientError::APIError {
                    code,
                    errors: vec![],
                }),
                "{code} must stay a failure"
            );
        }
    }

    /// Revocation addresses `secret/metadata/…`, not `secret/data/…`.
    ///
    /// The distinction is the whole point of the verb: the data path's delete
    /// leaves earlier versions readable, so revoking a secret that was ever
    /// rewritten there would leave the old value recoverable.
    ///
    /// Asserted on [`vaultrs`]'s own request type rather than on a call, since
    /// there is no store to call: it pins the endpoint
    /// `kv2::delete_metadata` targets, which is the string
    /// `swarm-bao.nix`'s grant has to match. A dependency bump that moved it
    /// would otherwise surface as a 403 at the first revocation, far from
    /// here.
    #[test]
    fn a_revocation_targets_every_version_and_not_just_the_newest() {
        use rustify::endpoint::Endpoint as _;

        let request = vaultrs::api::kv2::requests::DeleteSecretMetadataRequest::builder()
            .mount(MOUNT)
            .path("swarm/agents/atlas/queue")
            .build()
            .expect("the request builds");
        assert_eq!(request.path(), "secret/metadata/swarm/agents/atlas/queue");

        // The control: the verb deliberately not used, which addresses the
        // data path and soft-deletes exactly one version of it.
        let soft = vaultrs::api::kv2::requests::DeleteLatestSecretVersionRequest::builder()
            .mount(MOUNT)
            .path("swarm/agents/atlas/queue")
            .build()
            .expect("the request builds");
        assert_eq!(soft.path(), "secret/data/swarm/agents/atlas/queue");
        assert_ne!(request.path(), soft.path());
    }
}