gate ssh signing per agent: ssh permission, signing policy on the login role, ca key endpoint
This commit is contained in:
parent
aeb9317ad1
commit
910f9a6f50
1 changed files with 805 additions and 57 deletions
362
swarm-controller/src/agent_ssh.rs
Normal file
362
swarm-controller/src/agent_ssh.rs
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
//! SSH certificates for agents: the per-agent switch, and the public key of
|
||||
//! the CA that signs them.
|
||||
//!
|
||||
//! An agent may have its SSH key signed only while its `ssh` field in
|
||||
//! `agent-permissions` (`swarm_wire::agent_permissions`) is on. The grant is
|
||||
//! the policy [`policy::SSH_POLICY`], which holds `update` on the one sign path
|
||||
//! and nothing else. It is attached to the agent's cert-auth role
|
||||
//! `hive-agent-<name>` next to the shared policy while the field is on, and
|
||||
//! left off otherwise. The role is the state: [`crate::agent_identity`] keeps
|
||||
//! whatever the role has when it rewrites it for a new certificate, and
|
||||
//! [`reconcile`] is the only code that changes it.
|
||||
//!
|
||||
//! **When a change takes effect.** The role is rewritten when the field
|
||||
//! changes ([`SSH_CHANGED`]) and by a pass every [`RECONCILE_INTERVAL`]. A token
|
||||
//! carries the policies of the role it logged in under and keeps them until it
|
||||
//! expires, which is at most [`swarm_secret_client::client::CERT_ROLE_TOKEN_TTL`]
|
||||
//! after login, so a new login gets the new policies at once and an agent that
|
||||
//! was signed in when the switch went off can still sign for up to an hour.
|
||||
//! Switching on takes effect at the agent's next login, which its certificate
|
||||
//! unit makes on every run (every 20 minutes). A certificate already signed
|
||||
//! stays valid until it expires, at most three hours.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::StatusCode;
|
||||
use problem_details::ProblemDetails;
|
||||
use swarm_controller_api::ssh::SshCa;
|
||||
use swarm_queue_client::agent_permissions;
|
||||
use swarm_secret_client::{client::DEFAULT_CERT_MOUNT, policy};
|
||||
use swarm_wire::agent_permissions::AgentPermissions;
|
||||
|
||||
use crate::routes::{AppState, error_problem};
|
||||
use crate::schedules::read_or_default;
|
||||
use crate::store::AccessStore;
|
||||
|
||||
/// How often [`spawn`] re-checks every agent's role.
|
||||
const RECONCILE_INTERVAL: Duration = Duration::from_mins(5);
|
||||
|
||||
/// Woken when an agent's `ssh` field changes, so [`spawn`]'s pass applies it at
|
||||
/// once instead of at its next tick. A wake with no pass waiting is kept for
|
||||
/// the next one.
|
||||
pub(crate) static SSH_CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new();
|
||||
|
||||
/// The policies an agent's cert-auth role attaches: the shared one, and the
|
||||
/// signing grant when `ssh` is on.
|
||||
#[must_use]
|
||||
pub fn attached(ssh: bool) -> Vec<String> {
|
||||
let mut policies = vec![policy::SHARED_AGENT_POLICY.to_owned()];
|
||||
if ssh {
|
||||
policies.push(policy::SSH_POLICY.to_owned());
|
||||
}
|
||||
policies
|
||||
}
|
||||
|
||||
/// Whether `role`'s current policies include the signing grant, which is how
|
||||
/// [`crate::agent_identity`] knows to keep it when it rewrites the role.
|
||||
///
|
||||
/// # Errors
|
||||
/// The store could not be read.
|
||||
pub async fn role_has_ssh(
|
||||
store: &impl AccessStore,
|
||||
role: &str,
|
||||
) -> Result<bool, swarm_secret_client::Error> {
|
||||
Ok(store
|
||||
.cert_role_policies(DEFAULT_CERT_MOUNT, role)
|
||||
.await?
|
||||
.is_some_and(|p| p.iter().any(|p| p == policy::SSH_POLICY)))
|
||||
}
|
||||
|
||||
/// What one pass did, one count per role.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct Counts {
|
||||
pub rewritten: usize,
|
||||
pub already_current: usize,
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
/// Give every agent of `wanted` the policies its `ssh` field says, writing the
|
||||
/// signing policy first. An agent with no role is skipped: the mint that makes
|
||||
/// one keeps nothing, and the next pass attaches the grant.
|
||||
pub async fn reconcile(store: &impl AccessStore, wanted: &BTreeMap<String, bool>) -> Counts {
|
||||
let mut counts = Counts::default();
|
||||
if let Err(e) = store
|
||||
.write_policy(policy::SSH_POLICY, &policy::render_ssh_signer())
|
||||
.await
|
||||
{
|
||||
tracing::warn!(error = %e, "agent ssh: the signing policy was not written; retrying next pass");
|
||||
counts.failed = wanted.len();
|
||||
return counts;
|
||||
}
|
||||
for (agent, ssh) in wanted {
|
||||
let Ok(role) = policy::agent_object_name(agent) else {
|
||||
continue;
|
||||
};
|
||||
match reconcile_one(store, &role, *ssh).await {
|
||||
Ok(Some(true)) => counts.rewritten += 1,
|
||||
Ok(Some(false)) => counts.already_current += 1,
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
counts.failed += 1;
|
||||
tracing::warn!(role, error = %e, "agent ssh: this role's policies were not brought current");
|
||||
}
|
||||
}
|
||||
}
|
||||
if counts.rewritten > 0 || counts.failed > 0 {
|
||||
tracing::info!(
|
||||
rewritten = counts.rewritten,
|
||||
already_current = counts.already_current,
|
||||
failed = counts.failed,
|
||||
"agent ssh roles checked"
|
||||
);
|
||||
}
|
||||
counts
|
||||
}
|
||||
|
||||
/// `Some(true)` when the role was rewritten, `Some(false)` when it was already
|
||||
/// right, `None` when it does not exist.
|
||||
async fn reconcile_one(
|
||||
store: &impl AccessStore,
|
||||
role: &str,
|
||||
ssh: bool,
|
||||
) -> Result<Option<bool>, swarm_secret_client::Error> {
|
||||
let Some(mut current) = store.cert_role_policies(DEFAULT_CERT_MOUNT, role).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut wanted = attached(ssh);
|
||||
current.sort();
|
||||
wanted.sort();
|
||||
if current == wanted {
|
||||
return Ok(Some(false));
|
||||
}
|
||||
let policies: Vec<&str> = wanted.iter().map(String::as_str).collect();
|
||||
store
|
||||
.set_cert_role_policies(DEFAULT_CERT_MOUNT, role, &policies)
|
||||
.await?;
|
||||
tracing::info!(role, ssh, "agent ssh: role policies set");
|
||||
Ok(Some(true))
|
||||
}
|
||||
|
||||
/// `agent`'s `ssh` field: on only when its record says so.
|
||||
async fn ssh_of(store: &async_nats::jetstream::kv::Store, agent: &str) -> anyhow::Result<bool> {
|
||||
let (record, _, _) = read_or_default::<AgentPermissions>(store, agent).await?;
|
||||
Ok(record.ssh == Some(true))
|
||||
}
|
||||
|
||||
/// One pass: every agent holding a cert-auth role, with its field. An agent
|
||||
/// whose record cannot be read is left out, so its role stays as it is.
|
||||
async fn desired(
|
||||
queue: &async_nats::Client,
|
||||
roster: &[String],
|
||||
) -> anyhow::Result<BTreeMap<String, bool>> {
|
||||
let store = agent_permissions::open_or_create(queue).await?;
|
||||
let mut wanted = BTreeMap::new();
|
||||
for agent in roster {
|
||||
match ssh_of(&store, agent).await {
|
||||
Ok(on) => {
|
||||
wanted.insert(agent.clone(), on);
|
||||
}
|
||||
Err(e) => tracing::warn!(
|
||||
%agent,
|
||||
error = %format!("{e:#}"),
|
||||
"agent ssh: the agent's record was not read; leaving its role as it is"
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(wanted)
|
||||
}
|
||||
|
||||
/// Apply every agent's field now, again on each wake and on every tick. A pass
|
||||
/// that fails is logged and retried at the next tick.
|
||||
pub fn spawn(queue: async_nats::Client) {
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = ticker.tick() => {}
|
||||
() = SSH_CHANGED.notified() => {}
|
||||
}
|
||||
if let Err(e) = pass(&queue).await {
|
||||
tracing::warn!(error = %format!("{e:#}"), "agent ssh: pass failed; retrying next tick");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn pass(queue: &async_nats::Client) -> anyhow::Result<()> {
|
||||
use anyhow::Context as _;
|
||||
let store = crate::store::connect()
|
||||
.await
|
||||
.context("logging in to the swarm secret store")?;
|
||||
let roles = store
|
||||
.list_cert_roles(DEFAULT_CERT_MOUNT)
|
||||
.await
|
||||
.context("listing the store's cert-auth roles")?;
|
||||
let roster = policy::agents_from_role_names(&roles);
|
||||
let wanted = desired(queue, &roster).await?;
|
||||
reconcile(&store, &wanted).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/ssh/ca",
|
||||
responses(
|
||||
(status = 200, description = "the public key of the SSH CA that signs agents' \
|
||||
certificates", body = SshCa),
|
||||
(status = 404, description = "the store holds no SSH CA yet (problem+json)",
|
||||
body = String),
|
||||
(status = 502, description = "the store could not be read (problem+json)",
|
||||
body = String),
|
||||
),
|
||||
tag = "agents"
|
||||
)]
|
||||
pub(crate) async fn get_ssh_ca(
|
||||
State(_state): State<AppState>,
|
||||
) -> Result<Json<SshCa>, ProblemDetails> {
|
||||
let store = crate::store::connect().await.map_err(|e| {
|
||||
error_problem(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
&format!("logging in to the swarm secret store: {e}"),
|
||||
)
|
||||
})?;
|
||||
match store.ssh_ca_public_key(policy::SSH_MOUNT).await {
|
||||
Ok(Some(public_key)) => Ok(Json(SshCa { public_key })),
|
||||
Ok(None) => Err(error_problem(
|
||||
StatusCode::NOT_FOUND,
|
||||
"the swarm secret store holds no SSH CA yet",
|
||||
)),
|
||||
Err(e) => Err(error_problem(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
&format!("reading the SSH CA: {e}"),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use swarm_secret_client::policy;
|
||||
|
||||
use crate::store::fake::{FakeAccess, FakeRole};
|
||||
|
||||
use super::{Counts, attached, reconcile, role_has_ssh};
|
||||
|
||||
fn role(policies: &[&str]) -> FakeRole {
|
||||
FakeRole {
|
||||
certificate: "CA".to_owned(),
|
||||
common_name: "hive-agent-a".to_owned(),
|
||||
policies: policies.iter().map(|p| (*p).to_owned()).collect(),
|
||||
ttls: swarm_secret_client::client::CertRoleTokenTtls {
|
||||
token_ttl: swarm_secret_client::client::CERT_ROLE_TOKEN_TTL_SECS,
|
||||
token_max_ttl: swarm_secret_client::client::CERT_ROLE_TOKEN_TTL_SECS,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn wanted(of: &[(&str, bool)]) -> BTreeMap<String, bool> {
|
||||
of.iter().map(|(a, on)| ((*a).to_owned(), *on)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_signing_policy_is_attached_only_while_on() {
|
||||
assert_eq!(attached(false), ["hive-agent"]);
|
||||
assert_eq!(attached(true), ["hive-agent", "hive-agent-ssh"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn switching_on_attaches_the_policy_and_keeps_the_rest_of_the_role() {
|
||||
let before = role(&["hive-agent"]);
|
||||
let store = FakeAccess::default().with_role("hive-agent-a", before.clone());
|
||||
let counts = reconcile(&store, &wanted(&[("a", true)])).await;
|
||||
assert_eq!(
|
||||
counts,
|
||||
Counts {
|
||||
rewritten: 1,
|
||||
..Counts::default()
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
store.role("hive-agent-a"),
|
||||
Some(FakeRole {
|
||||
policies: vec!["hive-agent".to_owned(), policy::SSH_POLICY.to_owned()],
|
||||
..before
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.calls()
|
||||
.contains(&"write_policy hive-agent-ssh".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn switching_off_takes_the_policy_away_and_a_current_role_is_left_alone() {
|
||||
let store = FakeAccess::default()
|
||||
.with_role("hive-agent-a", role(&["hive-agent", "hive-agent-ssh"]))
|
||||
.with_role("hive-agent-b", role(&["hive-agent"]));
|
||||
let counts = reconcile(&store, &wanted(&[("a", false), ("b", false)])).await;
|
||||
assert_eq!(
|
||||
counts,
|
||||
Counts {
|
||||
rewritten: 1,
|
||||
already_current: 1,
|
||||
failed: 0
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
store.role("hive-agent-a").map(|r| r.policies),
|
||||
Some(vec!["hive-agent".to_owned()])
|
||||
);
|
||||
// The role already right is not written.
|
||||
let writes = store
|
||||
.calls()
|
||||
.into_iter()
|
||||
.filter(|c| c.starts_with("set_cert_role_policies"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(writes, ["set_cert_role_policies hive-agent-a hive-agent"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn flipping_twice_ends_where_it_started() {
|
||||
let store = FakeAccess::default().with_role("hive-agent-a", role(&["hive-agent"]));
|
||||
reconcile(&store, &wanted(&[("a", true)])).await;
|
||||
assert!(role_has_ssh(&store, "hive-agent-a").await.expect("reads"));
|
||||
reconcile(&store, &wanted(&[("a", false)])).await;
|
||||
assert!(!role_has_ssh(&store, "hive-agent-a").await.expect("reads"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_with_no_role_is_not_given_one() {
|
||||
let store = FakeAccess::default();
|
||||
let counts = reconcile(&store, &wanted(&[("gone", true)])).await;
|
||||
assert_eq!(counts, Counts::default());
|
||||
assert_eq!(store.role("hive-agent-gone"), None);
|
||||
assert!(
|
||||
!role_has_ssh(&store, "hive-agent-gone")
|
||||
.await
|
||||
.expect("reads")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_role_the_store_refuses_to_write_is_counted_and_the_rest_go_on() {
|
||||
let store = FakeAccess::default()
|
||||
.with_role("hive-agent-a", role(&["hive-agent"]))
|
||||
.with_role("hive-agent-b", role(&["hive-agent"]))
|
||||
.with_unwritable_role("hive-agent-a");
|
||||
let counts = reconcile(&store, &wanted(&[("a", true), ("b", true)])).await;
|
||||
assert_eq!(
|
||||
counts,
|
||||
Counts {
|
||||
rewritten: 1,
|
||||
already_current: 0,
|
||||
failed: 1
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue