Watch
0
0
Fork
You've already forked hyperhive
0

a removed hive's OIDC client secret is deleted with its other store credentials

This commit is contained in:
maia 2026-11-19 08:02:22 +01:00 • committed by mara
commit 2eacb83b21
5 changed files with 104 additions and 39 deletions

View file

@ -48,7 +48,7 @@ forge is re-minted within five minutes.
Removing a hive from `swarm.hives` revokes its credentials on the next
controller start: its forge token, its seat in the `hives` team, its token
secret in the store and the store login it uses. The forge account stays, so
secret and OIDC client secret in the store and the store login it uses. The forge account stays, so
putting the hive back in `swarm.hives` gives it new credentials within five
minutes of that start. The controller logs one `hive is not in the roster` line
per hive it cut off.

View file

@ -57,7 +57,10 @@ For each such hive, in this order:
1. delete its `swarm-hive` forge token,
2. remove its account from the `agent-configs/hives` team,
3. delete the store secret `swarm/hives/<hive>/forge/token`,
4. delete the cert-auth role `hive-<hive>`.
4. delete the store secret `swarm/hives/<hive>/oidc/client`, the hive's queue
client secret (authelia's client for the hive leaves the config on the
rebuild that drops it; the stored plaintext doesn't),
5. delete the cert-auth role `hive-<hive>`.
The role goes last because it marks that work remains: a step that fails
leaves the role, and the next pass runs every step again. A step that finds
@ -72,8 +75,10 @@ swarm-controller writes at start.
Nothing happens when the roster didn't load. An unset or unparseable
`SWARM_CONTROLLER_HIVES`, and a roster with no hive, read the same as a removal of every
hive, so the controller logs an error at start and skips the
cut-off. The attic pull token and the OIDC client secret under
`swarm/hives/<hive>/` stay, as does the forge account itself.
cut-off. The attic pull token under `swarm/hives/<hive>/` stays, as does the
forge account itself. The pull token is one swarm-wide token copied to each
hive; a copy a removed hive already holds keeps working, read-only, until someone rotates the
cache's signing key.
## Per-agent forge accounts

View file

@ -539,6 +539,10 @@ let
capabilities = ["delete"]
}
path "${credentialMountPath}/metadata/swarm/hives/+/oidc/client" {
capabilities = ["delete"]
}
path "${credentialMountPath}/data/swarm/hives/+/squid-selfcheck" {
capabilities = ["create", "read"]
}

View file

@ -1912,12 +1912,16 @@ let
&& lib.hasInfix "path \"secret/data/swarm/hives/+/squid-selfcheck\" {\n capabilities = [\"create\", \"read\"]\n}" s
&& !(lib.hasInfix "secret/data/swarm/hives/*" s)
# The only metadata it reaches under hives is the archive grants (list
# and delete) and the forge token's delete.
# and delete) and the forge token's and the OIDC client's delete.
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/\" {\n capabilities = [\"list\"]\n}" s
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/archive/+\" {\n capabilities = [\"delete\"]\n}" s
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/forge/token\" {\n capabilities = [\"delete\"]\n}" s
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/oidc/client\" {\n capabilities = [\"delete\"]\n}" s
&& lib.all (
g: lib.hasInfix "/archive/" g.path || g.path == "secret/metadata/swarm/hives/+/forge/token"
g:
lib.hasInfix "/archive/" g.path
|| g.path == "secret/metadata/swarm/hives/+/forge/token"
|| g.path == "secret/metadata/swarm/hives/+/oidc/client"
) (lib.filter (g: lib.hasPrefix "secret/metadata/swarm/hives" g.path) (grantsIn s));
}
{
@ -2577,11 +2581,12 @@ let
# roster through `delete_all_versions`, which addresses the metadata
# path. `+` is one segment, so the stanza reaches the `forge/token` leaf
# of every hive and no other key under `hives/`: not the attic facts, the
# squid credential, the OIDC client or the archive keys, which carry
# their own stanzas. Pinned as the whole stanza, so a `read` or `list`
# squid credential or the archive keys, which carry their own stanzas.
# `forge::hive_cutoff` deletes the OIDC client secret the same way, under
# a stanza of its own for that one leaf. Pinned as the whole stanza, so a `read` or `list`
# that would expose version history fails. The control is the archive
# revocation's metadata stanza, which the same search must find.
name = "the controller may delete a hive's forge token secret, and no other key of a hive's by that path";
name = "the controller may delete a hive's forge token secret and OIDC client secret, and no other key of a hive's by that path";
ok =
let
s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script;
@ -2592,7 +2597,8 @@ let
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/forge/*" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/attic" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/squid-selfcheck" s)
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc" s);
&& lib.hasInfix "path \"secret/metadata/swarm/hives/+/oidc/client\" {\n capabilities = [\"delete\"]\n}" s
&& !(lib.hasInfix "secret/metadata/swarm/hives/+/oidc/*" s);
}
{
# The swarm appservice token is a homeserver-admin credential. The

View file

@ -2,21 +2,21 @@
//!
//! A hive dropped from `swarm.hives` keeps everything the controller gave it:
//! the forge token (read on every config repo), the seat in the
//! `agent-configs/hives` team, the token secret and the cert-auth role it
//! logs in to the store under. [`cut_off_removed`] ends each, for every hive
//! that holds one of them and is not in the roster, in this order:
//! `agent-configs/hives` team, the token secret, the OIDC client secret and the
//! cert-auth role it logs in to the store under. [`cut_off_removed`] ends each,
//! for every hive that holds one of them and is not in the roster, in this
//! order:
//!
//! 1. the hive's `swarm-hive` forge token is deleted,
//! 2. its account leaves the `hives` team,
//! 3. the token secret at `swarm/hives/<hive>/forge/token` is deleted,
//! 4. the cert-auth role `hive-<hive>` is deleted.
//! 4. the OIDC client secret at `swarm/hives/<hive>/oidc/client` is deleted,
//! 5. the cert-auth role `hive-<hive>` is deleted.
//!
//! The role goes last because it is the marker that work is left: a failure
//! anywhere above it leaves the role, the next pass finds the hive again and
//! runs every step again. Each step finds nothing to do when it already ran.
//! The forge account stays, so a hive put back in the roster gets its token
//! and seat from [`super::hive_token`]'s pass and its role from the store
//! access written at boot.
//! The role goes last because it marks that work is left: a failure above it
//! leaves the role, and the next pass runs every step again. The forge account
//! stays, so a hive put back in the roster gets its token and seat from
//! [`super::hive_token`]'s pass and its role from the store access at boot.
//!
//! Nothing is cut off from a roster that did not load or that holds no hive
//! (`startup::HiveRoster::absence_means_removal`): an unreadable roster would
@ -25,15 +25,16 @@
//! `hive-` prefix and not the agents' `hive-agent-` one.
//!
//! The grants are the `auth/cert/certs/hive-*` stanza and the
//! `metadata/swarm/hives/+/forge/token` delete of `swarm-bao.nix`'s
//! `controllerPolicyText`.
//! `metadata/swarm/hives/+/forge/token` and `.../oidc/client` deletes of
//! `swarm-bao.nix`'s `controllerPolicyText`. The OIDC step reads nothing: its
//! delete is the same for a secret that is not there, so it runs every pass.
use std::collections::BTreeSet;
use std::future::Future;
use anyhow::{Context, Result};
use forgejo_api::structs::AdminSearchUsersQuery;
use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy};
use swarm_secret_client::{SecretStore, client::DEFAULT_CERT_MOUNT, forge, policy, queue};
use super::agent_token::is_not_found;
use super::hive_token::{HIVE_TOKEN_NAME, HIVES_TEAM, TeamSeen, is_hive_account};
@ -91,11 +92,14 @@ pub trait Revoke {
fn remove_team_seat(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
/// Delete the secret holding `hive`'s forge token; whether there was one.
fn delete_token_secret(&self, hive: &str) -> impl Future<Output = Result<bool>> + Send;
/// Delete the secret holding `hive`'s OIDC client secret; safe when there is
/// none.
fn delete_oidc_secret(&self, hive: &str) -> impl Future<Output = Result<()>> + Send;
/// Delete the cert-auth role `hive-<hive>`; safe when there is none.
fn delete_role(&self, hive: &str) -> impl Future<Output = Result<()>> + Send;
}
/// Run the four steps for `hive`, in order, stopping at the first that fails.
/// Run the five steps for `hive`, in order, stopping at the first that fails.
///
/// `has_role` is whether the role listing named it, which is what makes the
/// role count as removed.
@ -126,6 +130,10 @@ pub async fn cut_off(revoke: &impl Revoke, hive: &str, has_role: bool) -> Result
{
removed.push("token secret");
}
revoke
.delete_oidc_secret(hive)
.await
.context("deleting the OIDC client secret")?;
revoke
.delete_role(hive)
.await
@ -333,6 +341,14 @@ impl Revoke for Live<'_> {
Ok(held)
}
async fn delete_oidc_secret(&self, hive: &str) -> Result<()> {
let path = queue::hive_client_path(hive)?;
self.store
.delete_all_versions(&path)
.await
.with_context(|| format!("deleting {path}"))
}
async fn delete_role(&self, hive: &str) -> Result<()> {
let role = policy::hive_object_name(hive)?;
self.store
@ -395,6 +411,7 @@ mod tests {
token: BTreeSet<String>,
seat: BTreeSet<String>,
secret: BTreeSet<String>,
oidc: BTreeSet<String>,
role: BTreeSet<String>,
calls: Vec<String>,
}
@ -412,6 +429,7 @@ mod tests {
s.token.insert((*h).to_owned());
s.seat.insert((*h).to_owned());
s.secret.insert((*h).to_owned());
s.oidc.insert((*h).to_owned());
s.role.insert(format!("hive-{h}"));
}
}
@ -427,12 +445,13 @@ mod tests {
self.state.lock().unwrap().calls.clone()
}
fn holds(&self, hive: &str) -> [bool; 4] {
fn holds(&self, hive: &str) -> [bool; 5] {
let s = self.state.lock().unwrap();
[
s.token.contains(hive),
s.seat.contains(hive),
s.secret.contains(hive),
s.oidc.contains(hive),
s.role.contains(&format!("hive-{hive}")),
]
}
@ -474,6 +493,12 @@ mod tests {
Ok(self.state.lock().unwrap().secret.remove(hive))
}
async fn delete_oidc_secret(&self, hive: &str) -> Result<()> {
self.step("delete_oidc_secret", hive)?;
self.state.lock().unwrap().oidc.remove(hive);
Ok(())
}
async fn delete_role(&self, hive: &str) -> Result<()> {
self.step("delete_role", hive)?;
self.state
@ -525,42 +550,66 @@ mod tests {
}
#[tokio::test]
async fn a_removed_hive_loses_its_token_seat_secret_and_role_in_that_order() {
async fn a_removed_hive_loses_its_token_seat_secrets_and_role_in_that_order() {
let fake = Fake::holding(&["gone", "kept"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
assert_eq!(fake.holds("gone"), [false; 4], "{:?}", fake.calls());
assert_eq!(fake.holds("gone"), [false; 5], "{:?}", fake.calls());
assert_eq!(
fake.calls(),
[
"delete_token gone",
"remove_team_seat gone",
"delete_token_secret gone",
"delete_oidc_secret gone",
"delete_role gone",
]
);
}
/// The OIDC client secret goes with the rest, and a roster hive keeps its own.
#[tokio::test]
async fn a_removed_hives_oidc_client_secret_is_deleted_and_a_roster_hives_is_not() {
let fake = Fake::holding(&["gone", "kept"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
assert!(!fake.state.lock().unwrap().oidc.contains("gone"));
assert!(fake.state.lock().unwrap().oidc.contains("kept"));
assert!(
fake.calls().contains(&"delete_oidc_secret gone".to_owned()),
"{:?}",
fake.calls()
);
}
/// A hive that never had an OIDC secret is cut off all the same.
#[tokio::test]
async fn a_hive_with_no_oidc_client_secret_is_cut_off_without_error() {
let fake = Fake::holding(&["gone"]);
fake.state.lock().unwrap().oidc.clear();
cut_off_removed(&fake, &roster(&["kept"])).await;
assert_eq!(fake.holds("gone"), [false; 5], "{:?}", fake.calls());
}
#[tokio::test]
async fn a_hive_in_the_roster_is_untouched() {
let fake = Fake::holding(&["gone", "kept"]);
cut_off_removed(&fake, &roster(&["kept"])).await;
assert_eq!(fake.holds("kept"), [true; 4], "{:?}", fake.calls());
assert_eq!(fake.holds("kept"), [true; 5], "{:?}", fake.calls());
assert!(fake.calls().iter().all(|c| !c.ends_with("kept")));
// The control: the same pass did cut the removed one.
assert_eq!(fake.holds("gone"), [false; 4]);
assert_eq!(fake.holds("gone"), [false; 5]);
}
#[tokio::test]
async fn an_empty_roster_cuts_nothing_off() {
let fake = Fake::holding(&["a", "b"]);
cut_off_removed(&fake, &[]).await;
assert_eq!(fake.holds("a"), [true; 4]);
assert_eq!(fake.holds("b"), [true; 4]);
assert_eq!(fake.holds("a"), [true; 5]);
assert_eq!(fake.holds("b"), [true; 5]);
assert!(fake.calls().is_empty(), "{:?}", fake.calls());
// The control: a roster that names another hive cuts both.
cut_off_removed(&fake, &roster(&["c"])).await;
assert_eq!(fake.holds("a"), [false; 4]);
assert_eq!(fake.holds("b"), [false; 4]);
assert_eq!(fake.holds("a"), [false; 5]);
assert_eq!(fake.holds("b"), [false; 5]);
}
#[tokio::test]
@ -585,16 +634,17 @@ mod tests {
#[tokio::test]
async fn a_failed_step_stops_the_rest_and_keeps_the_role_for_the_next_pass() {
for (step, left) in [
("delete_token", [true, true, true, true]),
("remove_team_seat", [false, true, true, true]),
("delete_token_secret", [false, false, true, true]),
("delete_role", [false, false, false, true]),
("delete_token", [true, true, true, true, true]),
("remove_team_seat", [false, true, true, true, true]),
("delete_token_secret", [false, false, true, true, true]),
("delete_oidc_secret", [false, false, false, true, true]),
("delete_role", [false, false, false, false, true]),
] {
let broken = Fake::holding(&["gone"]).failing(step);
cut_off_removed(&broken, &roster(&["kept"])).await;
assert_eq!(broken.holds("gone"), left, "{step}: {:?}", broken.calls());
assert!(
broken.holds("gone")[3],
broken.holds("gone")[4],
"{step}: the role is the marker that work is left"
);
@ -605,7 +655,7 @@ mod tests {
failing: None,
};
cut_off_removed(&healthy, &roster(&["kept"])).await;
assert_eq!(healthy.holds("gone"), [false; 4], "{step}");
assert_eq!(healthy.holds("gone"), [false; 5], "{step}");
}
}