//! SSH certificates for agents: the per-agent switch, and the public key of //! the CA that signs them. //! //! An agent may have its SSH key signed only while its `ssh` field in //! `agent-permissions` (`swarm_wire::agent_permissions`) is on. The grant is //! the policy [`policy::SSH_POLICY`], which holds `update` on the one sign path //! and nothing else. It is attached to the agent's cert-auth role //! `hive-agent-` next to the shared policy while the field is on, and //! left off otherwise. The role is the state: [`crate::agent_identity`] keeps //! whatever the role has when it rewrites it for a new certificate, and //! [`reconcile`] is the only code that changes it. //! //! **When a change takes effect.** The role is rewritten when the field //! changes ([`SSH_CHANGED`]) and by a pass every [`RECONCILE_INTERVAL`]. A token //! carries the policies of the role it logged in under and keeps them until it //! expires, which is at most [`swarm_secret_client::client::CERT_ROLE_TOKEN_TTL`] //! after login, so a new login gets the new policies at once and an agent that //! was signed in when the switch went off can still sign for up to an hour. //! Switching on takes effect at the agent's next login, which its certificate //! unit makes on every run (every 20 minutes). A certificate already signed //! stays valid until it expires, at most three hours. use std::collections::BTreeMap; use std::time::Duration; use axum::Json; use axum::extract::State; use axum::http::StatusCode; use problem_details::ProblemDetails; use swarm_controller_api::ssh::SshCa; use swarm_queue_client::agent_permissions; use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, policy}; use swarm_wire::agent_permissions::AgentPermissions; use crate::routes::{AppState, error_problem}; use crate::schedules::read_or_default; use crate::store::AccessStore; /// How often [`spawn`] re-checks every agent's role. const RECONCILE_INTERVAL: Duration = Duration::from_mins(5); /// Woken when an agent's `ssh` field changes, so [`spawn`]'s pass applies it at /// once instead of at its next tick. A wake with no pass waiting is kept for /// the next one. pub(crate) static SSH_CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new(); /// The policies an agent's cert-auth role attaches: the shared one, and the /// signing grant when `ssh` is on. #[must_use] pub fn attached(ssh: bool) -> Vec { let mut policies = vec![policy::SHARED_AGENT_POLICY.to_owned()]; if ssh { policies.push(policy::SSH_POLICY.to_owned()); } policies } /// Whether `role`'s current policies include the signing grant, which is how /// [`crate::agent_identity`] knows to keep it when it rewrites the role. /// /// # Errors /// The store could not be read. pub async fn role_has_ssh( store: &impl AccessStore, role: &str, ) -> Result { Ok(store .cert_role_policies(DEFAULT_CERT_MOUNT, role) .await? .is_some_and(|p| p.iter().any(|p| p == policy::SSH_POLICY))) } /// What one pass did, one count per role. #[derive(Debug, Default, PartialEq, Eq)] pub struct Counts { pub rewritten: usize, pub already_current: usize, pub failed: usize, } /// Give every agent of `wanted` the policies its `ssh` field says, writing the /// signing policy first. An agent with no role is skipped: the mint that makes /// one keeps nothing, and the next pass attaches the grant. pub async fn reconcile(store: &impl AccessStore, wanted: &BTreeMap) -> Counts { let mut counts = Counts::default(); if let Err(e) = store .write_policy(policy::SSH_POLICY, &policy::render_ssh_signer()) .await { tracing::warn!(error = %e, "agent ssh: the signing policy was not written; retrying next pass"); counts.failed = wanted.len(); return counts; } for (agent, ssh) in wanted { let Ok(role) = policy::agent_object_name(agent) else { continue; }; match reconcile_one(store, &role, *ssh).await { Ok(Some(true)) => counts.rewritten += 1, Ok(Some(false)) => counts.already_current += 1, Ok(None) => {} Err(e) => { counts.failed += 1; tracing::warn!(role, error = %e, "agent ssh: this role's policies were not brought current"); } } } if counts.rewritten > 0 || counts.failed > 0 { tracing::info!( rewritten = counts.rewritten, already_current = counts.already_current, failed = counts.failed, "agent ssh roles checked" ); } counts } /// `Some(true)` when the role was rewritten, `Some(false)` when it was already /// right, `None` when it does not exist. async fn reconcile_one( store: &impl AccessStore, role: &str, ssh: bool, ) -> Result, swarm_secret_client::Error> { let Some(mut current) = store.cert_role_policies(DEFAULT_CERT_MOUNT, role).await? else { return Ok(None); }; let mut wanted = attached(ssh); current.sort(); wanted.sort(); if current == wanted { return Ok(Some(false)); } let policies: Vec<&str> = wanted.iter().map(String::as_str).collect(); store .set_cert_role_policies(DEFAULT_CERT_MOUNT, role, &policies) .await?; tracing::info!(role, ssh, "agent ssh: role policies set"); Ok(Some(true)) } /// `agent`'s `ssh` field: on only when its record says so. async fn ssh_of(store: &async_nats::jetstream::kv::Store, agent: &str) -> anyhow::Result { let (record, _, _) = read_or_default::(store, agent).await?; Ok(record.ssh == Some(true)) } /// One pass: every agent holding a cert-auth role, with its field. An agent /// whose record cannot be read is left out, so its role stays as it is. async fn desired( queue: &async_nats::Client, roster: &[String], ) -> anyhow::Result> { let store = agent_permissions::open_or_create(queue).await?; let mut wanted = BTreeMap::new(); for agent in roster { match ssh_of(&store, agent).await { Ok(on) => { wanted.insert(agent.clone(), on); } Err(e) => tracing::warn!( %agent, error = %format!("{e:#}"), "agent ssh: the agent's record was not read; leaving its role as it is" ), } } Ok(wanted) } /// Apply every agent's field now, again on each wake and on every tick. A pass /// that fails is logged and retried at the next tick. pub fn spawn(queue: async_nats::Client) { tokio::spawn(async move { let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); loop { tokio::select! { _ = ticker.tick() => {} () = SSH_CHANGED.notified() => {} } if let Err(e) = pass(&queue).await { tracing::warn!(error = %format!("{e:#}"), "agent ssh: pass failed; retrying next tick"); } } }); } async fn pass(queue: &async_nats::Client) -> anyhow::Result<()> { use anyhow::Context as _; let store = crate::store::connect() .await .context("logging in to the swarm secret store")?; let roles = store .list_cert_roles(DEFAULT_CERT_MOUNT) .await .context("listing the store's cert-auth roles")?; let roster = policy::agents_from_role_names(&roles); let wanted = desired(queue, &roster).await?; reconcile(&store, &wanted).await; Ok(()) } #[utoipa::path( get, path = "/api/ssh/ca", responses( (status = 200, description = "the public key of the SSH CA that signs agents' \ certificates", body = SshCa), (status = 404, description = "the store holds no SSH CA yet (problem+json)", body = String), (status = 502, description = "the store could not be read (problem+json)", body = String), ), tag = "agents" )] pub(crate) async fn get_ssh_ca( State(_state): State, ) -> Result, ProblemDetails> { let store = crate::store::connect().await.map_err(|e| { error_problem( StatusCode::BAD_GATEWAY, &format!("logging in to the swarm secret store: {e}"), ) })?; match store.ssh_ca_public_key(policy::SSH_MOUNT).await { Ok(Some(public_key)) => Ok(Json(SshCa { public_key })), Ok(None) => Err(error_problem( StatusCode::NOT_FOUND, "the swarm secret store holds no SSH CA yet", )), Err(e) => Err(error_problem( StatusCode::BAD_GATEWAY, &format!("reading the SSH CA: {e}"), )), } } #[cfg(test)] mod tests { use std::collections::BTreeMap; use swarm_secret_client::policy; use crate::store::fake::{FakeAccess, FakeRole}; use super::{Counts, attached, reconcile, role_has_ssh}; fn role(policies: &[&str]) -> FakeRole { FakeRole { certificate: "CA".to_owned(), common_name: "hive-agent-a".to_owned(), policies: policies.iter().map(|p| (*p).to_owned()).collect(), ttls: swarm_secret_client::client::CertRoleTokenTtls { token_ttl: swarm_secret_client::client::CERT_ROLE_TOKEN_TTL_SECS, token_max_ttl: swarm_secret_client::client::CERT_ROLE_TOKEN_TTL_SECS, }, } } fn wanted(of: &[(&str, bool)]) -> BTreeMap { of.iter().map(|(a, on)| ((*a).to_owned(), *on)).collect() } #[test] fn the_signing_policy_is_attached_only_while_on() { assert_eq!(attached(false), ["hive-agent"]); assert_eq!(attached(true), ["hive-agent", "hive-agent-ssh"]); } #[tokio::test] async fn switching_on_attaches_the_policy_and_keeps_the_rest_of_the_role() { let before = role(&["hive-agent"]); let store = FakeAccess::default().with_role("hive-agent-a", before.clone()); let counts = reconcile(&store, &wanted(&[("a", true)])).await; assert_eq!( counts, Counts { rewritten: 1, ..Counts::default() } ); assert_eq!( store.role("hive-agent-a"), Some(FakeRole { policies: vec!["hive-agent".to_owned(), policy::SSH_POLICY.to_owned()], ..before }) ); assert!( store .calls() .contains(&"write_policy hive-agent-ssh".to_owned()) ); } #[tokio::test] async fn switching_off_takes_the_policy_away_and_a_current_role_is_left_alone() { let store = FakeAccess::default() .with_role("hive-agent-a", role(&["hive-agent", "hive-agent-ssh"])) .with_role("hive-agent-b", role(&["hive-agent"])); let counts = reconcile(&store, &wanted(&[("a", false), ("b", false)])).await; assert_eq!( counts, Counts { rewritten: 1, already_current: 1, failed: 0 } ); assert_eq!( store.role("hive-agent-a").map(|r| r.policies), Some(vec!["hive-agent".to_owned()]) ); // The role already right is not written. let writes = store .calls() .into_iter() .filter(|c| c.starts_with("set_cert_role_policies")) .collect::>(); assert_eq!(writes, ["set_cert_role_policies hive-agent-a hive-agent"]); } #[tokio::test] async fn flipping_twice_ends_where_it_started() { let store = FakeAccess::default().with_role("hive-agent-a", role(&["hive-agent"])); reconcile(&store, &wanted(&[("a", true)])).await; assert!(role_has_ssh(&store, "hive-agent-a").await.expect("reads")); reconcile(&store, &wanted(&[("a", false)])).await; assert!(!role_has_ssh(&store, "hive-agent-a").await.expect("reads")); } #[tokio::test] async fn an_agent_with_no_role_is_not_given_one() { let store = FakeAccess::default(); let counts = reconcile(&store, &wanted(&[("gone", true)])).await; assert_eq!(counts, Counts::default()); assert_eq!(store.role("hive-agent-gone"), None); assert!( !role_has_ssh(&store, "hive-agent-gone") .await .expect("reads") ); } #[tokio::test] async fn a_role_the_store_refuses_to_write_is_counted_and_the_rest_go_on() { let store = FakeAccess::default() .with_role("hive-agent-a", role(&["hive-agent"])) .with_role("hive-agent-b", role(&["hive-agent"])) .with_unwritable_role("hive-agent-a"); let counts = reconcile(&store, &wanted(&[("a", true), ("b", true)])).await; assert_eq!( counts, Counts { rewritten: 1, already_current: 0, failed: 1 } ); } }