diff --git a/docs/2-handbook/swarm/forge.md b/docs/2-handbook/swarm/forge.md index fe3464f60..e2822f694 100644 --- a/docs/2-handbook/swarm/forge.md +++ b/docs/2-handbook/swarm/forge.md @@ -48,7 +48,7 @@ forge is re-minted within five minutes. Removing a hive from `swarm.hives` revokes its credentials on the next controller start: its forge token, its seat in the `hives` team, its token -secret in the store and the store login it uses. The forge account stays, so +secret and OIDC client secret in the store and the store login it uses. The forge account stays, so putting the hive back in `swarm.hives` gives it new credentials within five minutes of that start. The controller logs one `hive is not in the roster` line per hive it cut off. diff --git a/docs/4-implementation/swarm/forge-accounts.md b/docs/4-implementation/swarm/forge-accounts.md index bbc756017..ffc12837e 100644 --- a/docs/4-implementation/swarm/forge-accounts.md +++ b/docs/4-implementation/swarm/forge-accounts.md @@ -57,7 +57,10 @@ For each such hive, in this order: 1. delete its `swarm-hive` forge token, 2. remove its account from the `agent-configs/hives` team, 3. delete the store secret `swarm/hives//forge/token`, -4. delete the cert-auth role `hive-`. +4. delete the store secret `swarm/hives//oidc/client`, the hive's queue + client secret (authelia's client for the hive leaves the config on the + rebuild that drops it; the stored plaintext doesn't), +5. delete the cert-auth role `hive-`. The role goes last because it marks that work remains: a step that fails leaves the role, and the next pass runs every step again. A step that finds @@ -72,8 +75,10 @@ swarm-controller writes at start. Nothing happens when the roster didn't load. An unset or unparseable `SWARM_CONTROLLER_HIVES`, and a roster with no hive, read the same as a removal of every hive, so the controller logs an error at start and skips the -cut-off. The attic pull token and the OIDC client secret under -`swarm/hives//` stay, as does the forge account itself. +cut-off. The attic pull token under `swarm/hives//` stays, as does the +forge account itself. The pull token is one swarm-wide token copied to each +hive; a copy a removed hive already holds keeps working, read-only, until someone rotates the +cache's signing key. ## Per-agent forge accounts diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index b3ce70555..6f666f54a 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -539,6 +539,10 @@ let capabilities = ["delete"] } + path "${credentialMountPath}/metadata/swarm/hives/+/oidc/client" { + capabilities = ["delete"] + } + path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" { capabilities = ["create", "read"] } diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix index bd382a635..ceb151f75 100644 --- a/nix/module-eval/bao-grants.nix +++ b/nix/module-eval/bao-grants.nix @@ -1912,12 +1912,16 @@ let && lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s && !(lib.hasInfix "secret/data/swarm/hives/*" s) # The only metadata it reaches under hives is the archive grants (list - # and delete) and the forge token's delete. + # and delete) and the forge token's and the OIDC client's delete. && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/\" {\n capabilities = [\"list\"]\n}" s && lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s && lib.hasInfix "path \"secret/metadata/swarm/hives/+/forge/token\" {\n capabilities = [\"delete\"]\n}" s + && lib.hasInfix "path \"secret/metadata/swarm/hives/+/oidc/client\" {\n capabilities = [\"delete\"]\n}" s && lib.all ( - g: lib.hasInfix "/archive/" g.path || g.path == "secret/metadata/swarm/hives/+/forge/token" + g: + lib.hasInfix "/archive/" g.path + || g.path == "secret/metadata/swarm/hives/+/forge/token" + || g.path == "secret/metadata/swarm/hives/+/oidc/client" ) (lib.filter (g: lib.hasPrefix "secret/metadata/swarm/hives" g.path) (grantsIn s)); } { @@ -2577,11 +2581,12 @@ let # roster through `delete_all_versions`, which addresses the metadata # path. `+` is one segment, so the stanza reaches the `forge/token` leaf # of every hive and no other key under `hives/`: not the attic facts, the - # squid credential, the OIDC client or the archive keys, which carry - # their own stanzas. Pinned as the whole stanza, so a `read` or `list` + # squid credential or the archive keys, which carry their own stanzas. + # `forge::hive_cutoff` deletes the OIDC client secret the same way, under + # a stanza of its own for that one leaf. Pinned as the whole stanza, so a `read` or `list` # that would expose version history fails. The control is the archive # revocation's metadata stanza, which the same search must find. - name = "the controller may delete a hive's forge token secret, and no other key of a hive's by that path"; + name = "the controller may delete a hive's forge token secret and OIDC client secret, and no other key of a hive's by that path"; ok = let s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; @@ -2592,7 +2597,8 @@ let && !(lib.hasInfix "secret/metadata/swarm/hives/+/forge/*" s) && !(lib.hasInfix "secret/metadata/swarm/hives/+/attic" s) && !(lib.hasInfix "secret/metadata/swarm/hives/+/squid-selfcheck" s) - && !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc" s); + && lib.hasInfix "path \"secret/metadata/swarm/hives/+/oidc/client\" {\n capabilities = [\"delete\"]\n}" s + && !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc/*" s); } { # The swarm appservice token is a homeserver-admin credential. The diff --git a/swarm-controller/src/forge/hive_cutoff.rs b/swarm-controller/src/forge/hive_cutoff.rs index b7bb014ef..0bf53bb81 100644 --- a/swarm-controller/src/forge/hive_cutoff.rs +++ b/swarm-controller/src/forge/hive_cutoff.rs @@ -2,21 +2,21 @@ //! //! A hive dropped from `swarm.hives` keeps everything the controller gave it: //! the forge token (read on every config repo), the seat in the -//! `agent-configs/hives` team, the token secret and the cert-auth role it -//! logs in to the store under. [`cut_off_removed`] ends each, for every hive -//! that holds one of them and is not in the roster, in this order: +//! `agent-configs/hives` team, the token secret, the OIDC client secret and the +//! cert-auth role it logs in to the store under. [`cut_off_removed`] ends each, +//! for every hive that holds one of them and is not in the roster, in this +//! order: //! //! 1. the hive's `swarm-hive` forge token is deleted, //! 2. its account leaves the `hives` team, //! 3. the token secret at `swarm/hives//forge/token` is deleted, -//! 4. the cert-auth role `hive-` is deleted. +//! 4. the OIDC client secret at `swarm/hives//oidc/client` is deleted, +//! 5. the cert-auth role `hive-` is deleted. //! -//! The role goes last because it is the marker that work is left: a failure -//! anywhere above it leaves the role, the next pass finds the hive again and -//! runs every step again. Each step finds nothing to do when it already ran. -//! The forge account stays, so a hive put back in the roster gets its token -//! and seat from [`super::hive_token`]'s pass and its role from the store -//! access written at boot. +//! The role goes last because it marks that work is left: a failure above it +//! leaves the role, and the next pass runs every step again. The forge account +//! stays, so a hive put back in the roster gets its token and seat from +//! [`super::hive_token`]'s pass and its role from the store access at boot. //! //! Nothing is cut off from a roster that did not load or that holds no hive //! (`startup::HiveRoster::absence_means_removal`): an unreadable roster would @@ -25,15 +25,16 @@ //! `hive-` prefix and not the agents' `hive-agent-` one. //! //! The grants are the `auth/cert/certs/hive-*` stanza and the -//! `metadata/swarm/hives/+/forge/token` delete of `swarm-bao.nix`'s -//! `controllerPolicyText`. +//! `metadata/swarm/hives/+/forge/token` and `.../oidc/client` deletes of +//! `swarm-bao.nix`'s `controllerPolicyText`. The OIDC step reads nothing: its +//! delete is the same for a secret that is not there, so it runs every pass. use std::collections::BTreeSet; use std::future::Future; use anyhow::{Context, Result}; use forgejo_api::structs::AdminSearchUsersQuery; -use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy}; +use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy, queue}; use super::agent_token::is_not_found; use super::hive_token::{HIVE_TOKEN_NAME, HIVES_TEAM, TeamSeen, is_hive_account}; @@ -91,11 +92,14 @@ pub trait Revoke { fn remove_team_seat(&self, hive: &str) -> impl Future> + Send; /// Delete the secret holding `hive`'s forge token; whether there was one. fn delete_token_secret(&self, hive: &str) -> impl Future> + Send; + /// Delete the secret holding `hive`'s OIDC client secret; safe when there is + /// none. + fn delete_oidc_secret(&self, hive: &str) -> impl Future> + Send; /// Delete the cert-auth role `hive-`; safe when there is none. fn delete_role(&self, hive: &str) -> impl Future> + Send; } -/// Run the four steps for `hive`, in order, stopping at the first that fails. +/// Run the five steps for `hive`, in order, stopping at the first that fails. /// /// `has_role` is whether the role listing named it, which is what makes the /// role count as removed. @@ -126,6 +130,10 @@ pub async fn cut_off(revoke: &impl Revoke, hive: &str, has_role: bool) -> Result { removed.push("token secret"); } + revoke + .delete_oidc_secret(hive) + .await + .context("deleting the OIDC client secret")?; revoke .delete_role(hive) .await @@ -333,6 +341,14 @@ impl Revoke for Live<'_> { Ok(held) } + async fn delete_oidc_secret(&self, hive: &str) -> Result<()> { + let path = queue::hive_client_path(hive)?; + self.store + .delete_all_versions(&path) + .await + .with_context(|| format!("deleting {path}")) + } + async fn delete_role(&self, hive: &str) -> Result<()> { let role = policy::hive_object_name(hive)?; self.store @@ -395,6 +411,7 @@ mod tests { token: BTreeSet, seat: BTreeSet, secret: BTreeSet, + oidc: BTreeSet, role: BTreeSet, calls: Vec, } @@ -412,6 +429,7 @@ mod tests { s.token.insert((*h).to_owned()); s.seat.insert((*h).to_owned()); s.secret.insert((*h).to_owned()); + s.oidc.insert((*h).to_owned()); s.role.insert(format!("hive-{h}")); } } @@ -427,12 +445,13 @@ mod tests { self.state.lock().unwrap().calls.clone() } - fn holds(&self, hive: &str) -> [bool; 4] { + fn holds(&self, hive: &str) -> [bool; 5] { let s = self.state.lock().unwrap(); [ s.token.contains(hive), s.seat.contains(hive), s.secret.contains(hive), + s.oidc.contains(hive), s.role.contains(&format!("hive-{hive}")), ] } @@ -474,6 +493,12 @@ mod tests { Ok(self.state.lock().unwrap().secret.remove(hive)) } + async fn delete_oidc_secret(&self, hive: &str) -> Result<()> { + self.step("delete_oidc_secret", hive)?; + self.state.lock().unwrap().oidc.remove(hive); + Ok(()) + } + async fn delete_role(&self, hive: &str) -> Result<()> { self.step("delete_role", hive)?; self.state @@ -525,42 +550,66 @@ mod tests { } #[tokio::test] - async fn a_removed_hive_loses_its_token_seat_secret_and_role_in_that_order() { + async fn a_removed_hive_loses_its_token_seat_secrets_and_role_in_that_order() { let fake = Fake::holding(&["gone", "kept"]); cut_off_removed(&fake, &roster(&["kept"])).await; - assert_eq!(fake.holds("gone"), [false; 4], "{:?}", fake.calls()); + assert_eq!(fake.holds("gone"), [false; 5], "{:?}", fake.calls()); assert_eq!( fake.calls(), [ "delete_token gone", "remove_team_seat gone", "delete_token_secret gone", + "delete_oidc_secret gone", "delete_role gone", ] ); } + /// The OIDC client secret goes with the rest, and a roster hive keeps its own. + #[tokio::test] + async fn a_removed_hives_oidc_client_secret_is_deleted_and_a_roster_hives_is_not() { + let fake = Fake::holding(&["gone", "kept"]); + cut_off_removed(&fake, &roster(&["kept"])).await; + assert!(!fake.state.lock().unwrap().oidc.contains("gone")); + assert!(fake.state.lock().unwrap().oidc.contains("kept")); + assert!( + fake.calls().contains(&"delete_oidc_secret gone".to_owned()), + "{:?}", + fake.calls() + ); + } + + /// A hive that never had an OIDC secret is cut off all the same. + #[tokio::test] + async fn a_hive_with_no_oidc_client_secret_is_cut_off_without_error() { + let fake = Fake::holding(&["gone"]); + fake.state.lock().unwrap().oidc.clear(); + cut_off_removed(&fake, &roster(&["kept"])).await; + assert_eq!(fake.holds("gone"), [false; 5], "{:?}", fake.calls()); + } + #[tokio::test] async fn a_hive_in_the_roster_is_untouched() { let fake = Fake::holding(&["gone", "kept"]); cut_off_removed(&fake, &roster(&["kept"])).await; - assert_eq!(fake.holds("kept"), [true; 4], "{:?}", fake.calls()); + assert_eq!(fake.holds("kept"), [true; 5], "{:?}", fake.calls()); assert!(fake.calls().iter().all(|c| !c.ends_with("kept"))); // The control: the same pass did cut the removed one. - assert_eq!(fake.holds("gone"), [false; 4]); + assert_eq!(fake.holds("gone"), [false; 5]); } #[tokio::test] async fn an_empty_roster_cuts_nothing_off() { let fake = Fake::holding(&["a", "b"]); cut_off_removed(&fake, &[]).await; - assert_eq!(fake.holds("a"), [true; 4]); - assert_eq!(fake.holds("b"), [true; 4]); + assert_eq!(fake.holds("a"), [true; 5]); + assert_eq!(fake.holds("b"), [true; 5]); assert!(fake.calls().is_empty(), "{:?}", fake.calls()); // The control: a roster that names another hive cuts both. cut_off_removed(&fake, &roster(&["c"])).await; - assert_eq!(fake.holds("a"), [false; 4]); - assert_eq!(fake.holds("b"), [false; 4]); + assert_eq!(fake.holds("a"), [false; 5]); + assert_eq!(fake.holds("b"), [false; 5]); } #[tokio::test] @@ -585,16 +634,17 @@ mod tests { #[tokio::test] async fn a_failed_step_stops_the_rest_and_keeps_the_role_for_the_next_pass() { for (step, left) in [ - ("delete_token", [true, true, true, true]), - ("remove_team_seat", [false, true, true, true]), - ("delete_token_secret", [false, false, true, true]), - ("delete_role", [false, false, false, true]), + ("delete_token", [true, true, true, true, true]), + ("remove_team_seat", [false, true, true, true, true]), + ("delete_token_secret", [false, false, true, true, true]), + ("delete_oidc_secret", [false, false, false, true, true]), + ("delete_role", [false, false, false, false, true]), ] { let broken = Fake::holding(&["gone"]).failing(step); cut_off_removed(&broken, &roster(&["kept"])).await; assert_eq!(broken.holds("gone"), left, "{step}: {:?}", broken.calls()); assert!( - broken.holds("gone")[3], + broken.holds("gone")[4], "{step}: the role is the marker that work is left" ); @@ -605,7 +655,7 @@ mod tests { failing: None, }; cut_off_removed(&healthy, &roster(&["kept"])).await; - assert_eq!(healthy.holds("gone"), [false; 4], "{step}"); + assert_eq!(healthy.holds("gone"), [false; 5], "{step}"); } }