agent capabilities live in a swarm KV record, set from swarmctl
An agent's capabilities move from the hive's meta repo
(`meta/capabilities.json`, injected as `HIVE_CAPABILITIES`) into the
`capabilities` field of its key in a new `agent-permissions` KV bucket,
which swarm-controller alone writes.
- swarm-queue-client: `agent_permissions` (bucket, key, `AgentPermissions`,
direct-get read, the import subject and its wire types).
- swarm-controller: creates the bucket at start; `GET|PUT|DELETE
/api/agents/{name}/permissions` (deny_unknown_fields and unknown
capability ⇒ 422, duplicate ⇒ 400, agent must be declared by a hive);
answers each hive's one-shot import, writing a key only where there is
none and only for agents that hive's wanted state declares.
- swarmctl: `agent permissions get|set|clear`.
- swarm-nats-auth: a hive reads every key of the bucket and sends its
import on its own subject; the controller subscribes to the imports and
may answer them (`responses`).
- hive-c0re: reads each of its agents' keys every 30s into
/var/lib/hyperhive/swarm-agent-permissions.json and restarts an agent,
gracefully, when its capabilities change. Unset is the compiled-in
default (ruth: manage_root_agent; everyone else: none). A failed read
keeps the stored value. On first boot it seeds that file from
meta/capabilities.json, sends the grants to the controller, and deletes
the file once the controller answers.
Deleted: the capabilities.json reader/writer, `HIVE_CAPABILITIES` in the
rendered meta flake, ruth's capability seed, the capabilities half of the
permissions rebuild job (`PermPayload::Capabilities`/`Combined`), the
dashboard's capabilities table and `capabilities_changed` event, and the
now-unused `Capability::description`.
Part of #5170.
This commit is contained in:
parent
fcda1f4a04
commit
f862354e43
1 changed files with 1943 additions and 1146 deletions
295
hive-c0re/src/workers/swarm_permissions.rs
Normal file
295
hive-c0re/src/workers/swarm_permissions.rs
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
//! Reading the capabilities the swarm grants this hive's agents.
|
||||
//!
|
||||
//! swarm-controller keeps one key per agent in the `agent-permissions` bucket
|
||||
//! ([`swarm_queue_client::agent_permissions`]). Every [`READ_INTERVAL`] this
|
||||
//! reads the key of each agent this hive has and stores what it finds with
|
||||
//! [`crate::capabilities`], where the container's bind mounts are worked out.
|
||||
//! Those are fixed when the container starts, so an agent whose capabilities
|
||||
//! changed is restarted, after its current turn.
|
||||
//!
|
||||
//! A failed read, or a bucket that does not answer, leaves the stored
|
||||
//! permissions as they are and restarts nothing: dropping them would put an
|
||||
//! agent back on its default, which can be wider than what the swarm set.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use swarm_queue_client::agent_permissions::{self, AgentPermissions, Read};
|
||||
|
||||
use crate::capabilities::{self, Stored};
|
||||
use crate::coordinator::Coordinator;
|
||||
use crate::stats::sweep_health::{self, SweepHealth};
|
||||
|
||||
/// How often the permissions are read.
|
||||
pub const READ_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// At [`READ_INTERVAL`] this is ~90 seconds of failing reads.
|
||||
const FAILURES_BEFORE_BANNER: u32 = 3;
|
||||
|
||||
/// Send the meta repo's grants once, then read the permissions now and every
|
||||
/// [`READ_INTERVAL`] until shutdown.
|
||||
pub async fn follow(
|
||||
client: async_nats::Client,
|
||||
coord: Arc<Coordinator>,
|
||||
hive: String,
|
||||
mut shutdown: tokio::sync::watch::Receiver<bool>,
|
||||
) {
|
||||
let mut health = SweepHealth::new("swarm_permissions_read", "warn", FAILURES_BEFORE_BANNER);
|
||||
loop {
|
||||
// While the meta repo's grants are not in the swarm yet, an unset key
|
||||
// is not the swarm's answer, so nothing is read until they are.
|
||||
let pass = match super::permissions_import::send_pending(&client, &hive).await {
|
||||
Ok(()) => sync(&client).await,
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
match pass {
|
||||
Ok(changed) => {
|
||||
health.record_ok();
|
||||
restart(&coord, changed).await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "swarm permissions: the pass failed");
|
||||
let err = format!("{e:#}");
|
||||
health.record_err(|ctx| {
|
||||
let age = ctx.since_last_ok.map_or_else(
|
||||
|| "no success this session".to_owned(),
|
||||
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
|
||||
);
|
||||
format!(
|
||||
"syncing the swarm's agent permissions is failing ({} consecutive, \
|
||||
{age}) — agents keep the permissions last read: {err}",
|
||||
ctx.consecutive
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(READ_INTERVAL) => {}
|
||||
_ = shutdown.changed() => {
|
||||
tracing::info!("swarm permissions: shutdown signal received");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read every agent's key and store the result. Returns the agents whose
|
||||
/// capabilities changed.
|
||||
async fn sync(client: &async_nats::Client) -> Result<Vec<String>> {
|
||||
// An unconnected client does not fail a JetStream request, it hangs on it.
|
||||
swarm_queue_client::ensure_connected(client)?;
|
||||
let agents: Vec<String> = crate::lifecycle::agents_for_meta_listing()
|
||||
.await
|
||||
.context("enumerating this hive's agents")?
|
||||
.into_iter()
|
||||
.map(|spec| spec.name)
|
||||
.collect();
|
||||
sync_with(&agents, &capabilities::path(), |agent| async move {
|
||||
agent_permissions::read(client, &agent)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
.with_context(|| format!("reading the swarm's permissions for {agent}"))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// [`sync`] with the read and the file given. Nothing is written unless every
|
||||
/// read succeeds.
|
||||
async fn sync_with<F, Fut>(agents: &[String], path: &Path, read: F) -> Result<Vec<String>>
|
||||
where
|
||||
F: Fn(String) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<Read>>,
|
||||
{
|
||||
let mut read_now = Stored::new();
|
||||
for agent in agents {
|
||||
match read(agent.clone()).await? {
|
||||
Read::Current(record) => {
|
||||
if record != AgentPermissions::default() {
|
||||
read_now.insert(agent.clone(), record);
|
||||
}
|
||||
}
|
||||
Read::NoBucket => {
|
||||
tracing::debug!("swarm permissions: the agent-permissions bucket does not answer");
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
}
|
||||
}
|
||||
let stored = capabilities::read_at(path).context("reading the stored swarm permissions")?;
|
||||
if read_now == stored {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
capabilities::write_at(path, &read_now).context("storing the swarm permissions")?;
|
||||
tracing::info!(permissions = ?read_now, "swarm permissions: stored the swarm's permissions");
|
||||
Ok(agents
|
||||
.iter()
|
||||
.filter(|a| {
|
||||
capabilities::effective(a, stored.get(*a))
|
||||
!= capabilities::effective(a, read_now.get(*a))
|
||||
})
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Restart each running agent in `changed` once its current turn is done, so
|
||||
/// its container starts with the bind mounts its new capabilities give. A
|
||||
/// stopped agent gets them at its next start.
|
||||
async fn restart(coord: &Arc<Coordinator>, changed: Vec<String>) {
|
||||
let mut running = Vec::new();
|
||||
for agent in changed {
|
||||
if crate::lifecycle::is_running(&agent).await {
|
||||
running.push(agent);
|
||||
}
|
||||
}
|
||||
if running.is_empty() {
|
||||
return;
|
||||
}
|
||||
tracing::info!(agents = ?running, "swarm permissions: capabilities changed, restarting");
|
||||
if let Err(e) = crate::job_queue::power::restart_many(coord, &running, true).await {
|
||||
tracing::error!(agents = ?running, error = ?e, "swarm permissions: restart insert failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use super::{Read, sync_with};
|
||||
use swarm_queue_client::agent_permissions::AgentPermissions;
|
||||
|
||||
fn record(json: &str) -> AgentPermissions {
|
||||
serde_json::from_str(json).expect("decodes")
|
||||
}
|
||||
|
||||
fn agents() -> Vec<String> {
|
||||
vec!["atlas".to_owned(), "ruth".to_owned()]
|
||||
}
|
||||
|
||||
/// Sync `agents()` against `path` with each agent read as `answer` gives.
|
||||
async fn sync(
|
||||
path: &Path,
|
||||
answer: impl Fn(&str) -> anyhow::Result<Read>,
|
||||
) -> anyhow::Result<Vec<String>> {
|
||||
sync_with(&agents(), path, |agent| {
|
||||
let out = answer(&agent);
|
||||
async move { out }
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// No key anywhere is everyone on the default: nothing stored, and no
|
||||
/// restart, since that is what the agents already run with.
|
||||
#[tokio::test]
|
||||
async fn missing_keys_are_the_default_and_restart_nothing() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("p.json");
|
||||
let changed = sync(&path, |_| Ok(Read::Current(AgentPermissions::default())))
|
||||
.await
|
||||
.expect("read");
|
||||
assert!(changed.is_empty());
|
||||
assert!(
|
||||
crate::capabilities::read_at(&path)
|
||||
.expect("read")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
crate::capabilities::holds(
|
||||
&path,
|
||||
"ruth",
|
||||
hive_sh4re::permissions::Capability::ManageRootAgent
|
||||
)
|
||||
.expect("read")
|
||||
);
|
||||
}
|
||||
|
||||
/// Revoking ruth's default, and granting atlas, restarts both; clearing
|
||||
/// them again restarts both back.
|
||||
#[tokio::test]
|
||||
async fn a_changed_capability_asks_for_a_restart() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("p.json");
|
||||
let changed = sync(&path, |agent| {
|
||||
Ok(Read::Current(record(if agent == "ruth" {
|
||||
r#"{"capabilities":[]}"#
|
||||
} else {
|
||||
r#"{"capabilities":["manage_root_agent"]}"#
|
||||
})))
|
||||
})
|
||||
.await
|
||||
.expect("read");
|
||||
assert_eq!(changed, agents());
|
||||
|
||||
// The same answer again changes nothing.
|
||||
let unchanged = sync(&path, |agent| {
|
||||
Ok(Read::Current(record(if agent == "ruth" {
|
||||
r#"{"capabilities":[]}"#
|
||||
} else {
|
||||
r#"{"capabilities":["manage_root_agent"]}"#
|
||||
})))
|
||||
})
|
||||
.await
|
||||
.expect("read");
|
||||
assert!(unchanged.is_empty());
|
||||
|
||||
let cleared = sync(&path, |_| Ok(Read::Current(AgentPermissions::default())))
|
||||
.await
|
||||
.expect("read");
|
||||
assert_eq!(cleared, agents());
|
||||
}
|
||||
|
||||
/// Setting ruth's default explicitly is a different record but the same
|
||||
/// mounts, so it is stored and restarts nothing.
|
||||
#[tokio::test]
|
||||
async fn an_explicit_default_is_stored_without_a_restart() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("p.json");
|
||||
let changed = sync(&path, |agent| {
|
||||
Ok(Read::Current(if agent == "ruth" {
|
||||
record(r#"{"capabilities":["manage_root_agent"]}"#)
|
||||
} else {
|
||||
AgentPermissions::default()
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.expect("read");
|
||||
assert!(changed.is_empty());
|
||||
assert!(
|
||||
crate::capabilities::read_at(&path)
|
||||
.expect("read")
|
||||
.contains_key("ruth")
|
||||
);
|
||||
}
|
||||
|
||||
/// One failed read, or a bucket that does not answer, keeps what was
|
||||
/// stored and restarts nothing: never a fall back to the default.
|
||||
#[tokio::test]
|
||||
async fn a_failed_read_keeps_the_last_value() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("p.json");
|
||||
let revoked = || Ok(Read::Current(record(r#"{"capabilities":[]}"#)));
|
||||
// An empty list is atlas's default already, so only ruth changes.
|
||||
assert_eq!(
|
||||
sync(&path, |_| revoked()).await.expect("read"),
|
||||
vec!["ruth".to_owned()]
|
||||
);
|
||||
let before = std::fs::read(&path).expect("stored");
|
||||
|
||||
sync(&path, |agent| {
|
||||
if agent == "ruth" {
|
||||
Err(anyhow::anyhow!("timed out"))
|
||||
} else {
|
||||
Ok(Read::Current(AgentPermissions::default()))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect_err("a failed read fails the pass");
|
||||
assert_eq!(std::fs::read(&path).expect("stored"), before);
|
||||
|
||||
let changed = sync(&path, |_| Ok(Read::NoBucket))
|
||||
.await
|
||||
.expect("no bucket");
|
||||
assert!(changed.is_empty());
|
||||
assert_eq!(std::fs::read(&path).expect("stored"), before);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue