mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

controller gates logs and metrics on the hive capability and mirrors the grant

Browse source
This commit is contained in:
atlas 2026-11-01 17:40:09 +01:00
commit e7e3281098
1 changed files with 519 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

203
swarm-controller/src/agent_permissions.rs
View file

@ -23,11 +23,13 @@ use problem_details::ProblemDetails;
use serde::Serialize;
use swarm_controller_api::permissions::Permissions;
use swarm_queue_client::agent_permissions;
use swarm_secret_client::observability::Grant;
use swarm_wire::agent_permissions::{
AgentPermissions, Import, ImportReply, hive_from_import_subject, import_subject_wildcard,
};
use utoipa::ToSchema;
use crate::observability_grant::{self, grant_of, mirror, require_tolerant_hive};
use crate::routes::{AppState, error_problem, wanted_error_status};
use crate::schedules::{live, queue, read_or_default, refused, write, write_revision};
use crate::wanted::WantedWriter;
@ -102,6 +104,31 @@ fn logged(op: &'static str, agent: &str, problem: ProblemDetails) -> ProblemDeta
problem
}
/// Bring the store's mirror of `agent`'s `logs`/`metrics` grant in line with
/// a record that went from `before` to `after`, after the record is written.
/// A failure is an error to the caller, who repeats the request: `mirror`
/// rewrites whenever either record holds a grant.
async fn mirrored(
agent: &str,
before: &AgentPermissions,
after: &AgentPermissions,
) -> Result<(), ProblemDetails> {
let held = |v: &AgentPermissions| v.permissions.clone().unwrap_or_default();
mirror(&observability_grant::Store, agent, &held(before), &held(after))
.await
.map_err(|e| {
tracing::warn!(agent, error = %format!("{e:#}"), "agent permissions: mirroring the grant failed");
error_problem(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("the permissions are stored, but mirroring logs/metrics to the secret store failed: {e:#}"),
)
})
}
fn hive_names(state: &AppState) -> Vec<String> {
state.hives.iter().map(|h| h.name.clone()).collect()
}
fn queue_problem(e: &anyhow::Error) -> ProblemDetails {
tracing::warn!(error = %format!("{e:#}"), "agent permissions: queue failure");
error_problem(wanted_error_status(e), &format!("{e:#}"))
@ -185,6 +212,19 @@ pub(crate) async fn put_agent_permissions(
&format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"),
)));
}
if let Some(perms) = &req.permissions
&& grant_of(perms) != Grant::default()
{
let (Some(wanted), hives) = (state.wanted.as_ref(), hive_names(&state)) else {
return Err(refuse(error_problem(
StatusCode::SERVICE_UNAVAILABLE,
"no swarm queue is configured on this host",
)));
};
require_tolerant_hive(&client, wanted, &hives, &agent)
.await
.map_err(|why| refuse(error_problem(StatusCode::CONFLICT, &why)))?;
}
let store = open_or_create(&client).await?;
for _ in 0..MAX_ATTEMPTS {
let (current, revision, undecodable) = read_or_default::<AgentPermissions>(&store, &agent)
@ -200,6 +240,7 @@ pub(crate) async fn put_agent_permissions(
.map_err(|e| queue_problem(&e))?
{
tracing::info!(%agent, ?value, "set agent permissions");
mirrored(&agent, &current, &value).await?;
return Ok(Json(view(value)));
}
}
@ -255,6 +296,7 @@ pub(crate) async fn delete_agent_permissions(
continue;
}
tracing::info!(%agent, "cleared agent permissions");
mirrored(&agent, &current, &kept).await?;
return Ok(StatusCode::NO_CONTENT);
}
Err(queue_problem(&anyhow::anyhow!(
@ -271,10 +313,17 @@ pub(crate) async fn delete_agent_permissions(
pub(crate) async fn clear_on_destroy(state: &AppState, hive: &str, agent: &str) {
let cleared = async {
let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?;
agent_permissions::open_or_create(&client)
.await?
.delete(agent)
.await?;
let store = agent_permissions::open_or_create(&client).await?;
let (current, _, _) = read_or_default::<AgentPermissions>(&store, agent).await?;
store.delete(agent).await?;
// The mirrored grant goes with the record.
mirror(
&observability_grant::Store,
agent,
&current.permissions.unwrap_or_default(),
&[],
)
.await?;
anyhow::Ok(())
}
.await;
@ -457,10 +506,15 @@ pub(crate) fn spawn(client: async_nats::Client, wanted: Arc<WantedWriter>, hives
return;
};
tokio::spawn({
let (store, wanted, hives) = (store.clone(), Arc::clone(&wanted), hives.clone());
let (client, store, wanted, hives) = (
client.clone(),
store.clone(),
Arc::clone(&wanted),
hives.clone(),
);
async move {
// Retries without end, so it answers only once it has worked.
let _ = (|| seed_ruth_grant(&store, &wanted, &hives))
let _ = (|| seed_ruth_grant(&client, &store, &wanted, &hives))
.retry(every_five_seconds())
.notify(|e: &anyhow::Error, _| {
tracing::warn!(
@ -539,6 +593,7 @@ const SEEDED_AGENT: &str = "ruth";
/// `ruth` whose permissions were unset; this writes that grant into her key
/// so she keeps her recovery mounts. Idempotent: a set key is never touched.
async fn seed_ruth_grant(
client: &async_nats::Client,
store: &async_nats::jetstream::kv::Store,
wanted: &WantedWriter,
hives: &[String],
@ -549,16 +604,27 @@ async fn seed_ruth_grant(
declarations.push(d);
}
}
seed_into(store, &crate::agent_renewal::live_agents(&declarations)).await?;
Ok(())
let declared = crate::agent_renewal::live_agents(&declarations);
if !declared.contains(SEEDED_AGENT) {
return Ok(());
}
let tolerant =
observability_grant::hive_is_tolerant(client, wanted, hives, SEEDED_AGENT).await?;
seed_into(store, &declared, tolerant).await?;
// At every start, so a mirror that failed to write is retried.
let (current, _, _) = read_or_default::<AgentPermissions>(store, SEEDED_AGENT).await?;
let held = current.permissions.unwrap_or_default();
mirror(&observability_grant::Store, SEEDED_AGENT, &[], &held).await
}
/// Write [`Permission::DEFAULT`] plus `manage_root_agent` under
/// [`SEEDED_AGENT`]'s key when `declared` holds her and her `permissions`
/// are unset (no key, a deleted key, or `{}`). `true` when it wrote.
/// Write [`Permission::DEFAULT`] plus `manage_root_agent`, and `logs` and
/// `metrics` when her hive is `tolerant`, under [`SEEDED_AGENT`]'s key when
/// `declared` holds her and her `permissions` are unset (no key, a deleted
/// key, or `{}`). `true` when it wrote.
async fn seed_into(
store: &async_nats::jetstream::kv::Store,
declared: &BTreeSet<String>,
tolerant: bool,
) -> anyhow::Result<bool> {
if !declared.contains(SEEDED_AGENT) {
return Ok(false);
@ -570,6 +636,9 @@ async fn seed_into(
}
let mut perms = Permission::DEFAULT.to_vec();
perms.push(Permission::ManageRootAgent);
if tolerant {
perms.extend([Permission::Logs, Permission::Metrics]);
}
let value = AgentPermissions {
permissions: Some(perms),
..AgentPermissions::default()
@ -616,7 +685,7 @@ async fn import_into(
}
#[cfg(test)]
mod tests {
pub(crate) mod tests {
use super::{
Permissions, delete_agent_permissions, get_agent_permissions, import_into,
put_agent_permissions, seed_into,
@ -701,7 +770,7 @@ mod tests {
);
}
async fn published(client: &async_nats::Client, agent: &str) -> Option<String> {
pub(crate) async fn published(client: &async_nats::Client, agent: &str) -> Option<String> {
let store = swarm_queue_client::agent_permissions::open_or_create(client)
.await
.expect("bucket");
@ -945,6 +1014,91 @@ mod tests {
assert_eq!(published(&client, "atlas").await.as_deref(), Some(older));
}
async fn publish_snapshot(client: &async_nats::Client, json: &str) {
swarm_queue_client::status::open_or_create(client)
.await
.expect("bucket")
.put("pr1ma", json.as_bytes().to_vec().into())
.await
.expect("snapshot");
}
fn observing(perms: &[Permission]) -> Permissions {
Permissions {
permissions: Some(perms.to_vec()),
unrestricted_internet: None,
internet_log: None,
}
}
/// `logs` and `metrics` are refused, naming the hive and writing nothing,
/// while the agent's hive has published no snapshot or one without the
/// capability. Control: a grant that names neither passes the same hive.
#[tokio::test]
async fn logs_and_metrics_need_a_hive_that_advertises_tolerant_decoding() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
for perm in [Permission::Logs, Permission::Metrics] {
let none = put(&state, "atlas", observing(&[perm]))
.await
.expect_err("no snapshot");
assert_eq!(none.status, Some(StatusCode::CONFLICT));
assert!(none.detail.as_deref().unwrap_or_default().contains("pr1ma"));
}
publish_snapshot(&client, r#"{"status":"ok","warnings":[]}"#).await;
let old = put(&state, "atlas", observing(&[Permission::Logs]))
.await
.expect_err("old hive");
assert_eq!(old.status, Some(StatusCode::CONFLICT));
assert_eq!(published(&client, "atlas").await, None, "nothing written");
let _ = put(&state, "atlas", observing(&[Permission::Messaging]))
.await
.expect("control: no log or metric grant");
}
/// With the capability advertised the gate passes: the record is written,
/// and the mirror (which needs the secret store, absent here) is what
/// fails.
#[tokio::test]
async fn an_advertising_hive_passes_the_gate() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
publish_snapshot(&client, r#"{"capabilities":["tolerant-permissions"]}"#).await;
let problem = put(&state, "atlas", observing(&[Permission::Metrics]))
.await
.expect_err("no secret store to mirror to");
assert_eq!(problem.status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["metrics"]}"#)
);
}
/// With her hive tolerant ruth is seeded with `logs` and `metrics`; with
/// an old hive she gets the set she always had.
#[tokio::test]
async fn the_seed_includes_logs_and_metrics_on_a_tolerant_hive() {
let nats = TestNats::start();
let client = nats.client().await;
let store = swarm_queue_client::agent_permissions::open_or_create(&client)
.await
.expect("bucket");
let declared = ["ruth".to_owned()].into();
assert!(seed_into(&store, &declared, true).await.expect("seeded"));
assert_eq!(
published(&client, "ruth").await.as_deref(),
Some(
r#"{"permissions":["messaging","meta","inbox","execution","manage_root_agent","logs","metrics"]}"#
)
);
}
/// A name no hive declares, or a name that is not an identifier, is a 400
/// and writes nothing.
#[tokio::test]
@ -1170,25 +1324,29 @@ mod tests {
.expect("bucket");
let declared = ["atlas".to_owned(), "ruth".to_owned()].into();
assert!(seed_into(&store, &declared).await.expect("seeded"));
assert!(seed_into(&store, &declared, false).await.expect("seeded"));
assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED));
assert_eq!(published(&client, "atlas").await, None, "only ruth");
assert!(!seed_into(&store, &declared).await.expect("idempotent"));
assert!(
!seed_into(&store, &declared, false)
.await
.expect("idempotent")
);
store.delete("ruth").await.expect("delete");
assert!(seed_into(&store, &declared).await.expect("seeded"));
assert!(seed_into(&store, &declared, false).await.expect("seeded"));
assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED));
store
.put("ruth", b"{}".to_vec().into())
.await
.expect("unset");
assert!(seed_into(&store, &declared).await.expect("seeded"));
assert!(seed_into(&store, &declared, false).await.expect("seeded"));
assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED));
}
/// A ruth key the swarm already set, even to nothing, is left as it is;
/// so is one this build cannot decode.
/// so is one holding only a name this build does not know.
#[tokio::test]
async fn the_seed_leaves_a_set_ruth_key_alone() {
let nats = TestNats::start();
@ -1206,7 +1364,10 @@ mod tests {
.put("ruth", set.as_bytes().to_vec().into())
.await
.expect("set");
assert!(!seed_into(&store, &declared).await.expect("read"), "{set}");
assert!(
!seed_into(&store, &declared, false).await.expect("read"),
"{set}"
);
assert_eq!(published(&client, "ruth").await.as_deref(), Some(set));
}
}
@ -1222,12 +1383,12 @@ mod tests {
.expect("bucket");
let declared = ["atlas".to_owned()].into();
assert!(!seed_into(&store, &declared).await.expect("read"));
assert!(!seed_into(&store, &declared, false).await.expect("read"));
assert_eq!(published(&client, "ruth").await, None);
assert_eq!(published(&client, "atlas").await, None);
let declared = ["atlas".to_owned(), "ruth".to_owned()].into();
assert!(seed_into(&store, &declared).await.expect("seeded"));
assert!(seed_into(&store, &declared, false).await.expect("seeded"));
assert_eq!(published(&client, "ruth").await.as_deref(), Some(SEEDED));
}