mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

order grant mirror writes by direction, re-mirror every five minutes, gate on the agent's own harness

Browse source
This commit is contained in:
atlas 2026-11-01 18:27:55 +01:00
commit 329d556ab9
1 changed files with 793 additions and 118 deletions
Download patch file Download diff file Expand all files Collapse all files

493
swarm-controller/src/agent_permissions.rs
View file

@ -8,7 +8,7 @@
//! restarts it when `manage_root_agent` changes. An unset field leaves the
//! agent on the compiled-in default; an empty list grants nothing.
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use std::time::Duration;
@ -29,7 +29,9 @@ use swarm_wire::agent_permissions::{
};
use utoipa::ToSchema;
use crate::observability_grant::{self, grant_of, mirror, require_tolerant_hive};
use crate::observability_grant::{
self, GrantLeaves, adds, grant_of, mirror, require_tolerant_agent, revokes,
};
use crate::routes::{AppState, error_problem, wanted_error_status};
use crate::schedules::{live, queue, read_or_default, refused, write, write_revision};
use crate::wanted::WantedWriter;
@ -104,25 +106,51 @@ fn logged(op: &'static str, agent: &str, problem: ProblemDetails) -> ProblemDeta
problem
}
/// Bring the store's mirror of `agent`'s `logs`/`metrics` grant in line with
/// a record that went from `before` to `after`, after the record is written.
/// A failure is an error to the caller, who repeats the request: `mirror`
/// rewrites whenever either record holds a grant.
async fn mirrored(
/// The permissions a record holds.
fn held(record: &AgentPermissions) -> Vec<Permission> {
record.permissions.clone().unwrap_or_default()
}
/// The error for a mirror write that failed. `stored` says which side of the
/// change landed first: a revoke writes the leaf before the record, so the
/// record is untouched and the revoke did not happen; a grant writes the
/// record first, so the gateway keeps denying until the request is repeated
/// or the next pass of [`spawn`] writes the leaf.
fn mirror_problem(agent: &str, e: &anyhow::Error, stored: bool) -> ProblemDetails {
tracing::warn!(agent, error = %format!("{e:#}"), stored, "agent permissions: mirroring the grant failed");
let outcome = if stored {
"the permissions are stored, but the log and metric gateways deny until the grant is mirrored"
} else {
"nothing was changed"
};
error_problem(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("mirroring logs/metrics to the secret store failed; {outcome}: {e:#}"),
)
}
/// A 409 unless `agent` may be granted `logs` or `metrics`, see
/// [`require_tolerant_agent`].
async fn require_tolerant_for(
state: &AppState,
client: &async_nats::Client,
agent: &str,
before: &AgentPermissions,
after: &AgentPermissions,
) -> Result<(), ProblemDetails> {
let held = |v: &AgentPermissions| v.permissions.clone().unwrap_or_default();
mirror(&observability_grant::Store, agent, &held(before), &held(after))
.await
.map_err(|e| {
tracing::warn!(agent, error = %format!("{e:#}"), "agent permissions: mirroring the grant failed");
error_problem(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("the permissions are stored, but mirroring logs/metrics to the secret store failed: {e:#}"),
)
})
let Some(wanted) = state.wanted.as_ref() else {
return Err(error_problem(
StatusCode::SERVICE_UNAVAILABLE,
"no swarm queue is configured on this host",
));
};
require_tolerant_agent(
client,
wanted,
&hive_names(state),
state.agent_login.as_deref(),
agent,
)
.await
.map_err(|why| error_problem(StatusCode::CONFLICT, &why))
}
fn hive_names(state: &AppState) -> Vec<String> {
@ -197,6 +225,22 @@ pub(crate) async fn put_agent_permissions(
State(state): State<AppState>,
Path(name): Path<String>,
req: Result<Json<Permissions>, JsonRejection>,
) -> Result<Json<Permissions>, ProblemDetails> {
put_with(&observability_grant::Store, state, name, req).await
}
/// [`put_agent_permissions`] against `leaves`.
///
/// The record and the leaf are written in the order that fails towards
/// denying: a change that drops `logs` or `metrics` writes the leaf first, so a
/// failed write leaves both granted and the request an honest error; a change
/// that adds one writes the record first, so a failed write leaves the gateway
/// denying.
async fn put_with(
leaves: &impl GrantLeaves,
state: AppState,
name: String,
req: Result<Json<Permissions>, JsonRejection>,
) -> Result<Json<Permissions>, ProblemDetails> {
let refuse = |p| logged("put", &name, p);
let Json(req) =
@ -212,19 +256,6 @@ pub(crate) async fn put_agent_permissions(
&format!("no hive's wanted state declares an agent {agent:?} that is not destroyed"),
)));
}
if let Some(perms) = &req.permissions
&& grant_of(perms) != Grant::default()
{
let (Some(wanted), hives) = (state.wanted.as_ref(), hive_names(&state)) else {
return Err(refuse(error_problem(
StatusCode::SERVICE_UNAVAILABLE,
"no swarm queue is configured on this host",
)));
};
require_tolerant_hive(&client, wanted, &hives, &agent)
.await
.map_err(|why| refuse(error_problem(StatusCode::CONFLICT, &why)))?;
}
let store = open_or_create(&client).await?;
for _ in 0..MAX_ATTEMPTS {
let (current, revision, undecodable) = read_or_default::<AgentPermissions>(&store, &agent)
@ -234,13 +265,29 @@ pub(crate) async fn put_agent_permissions(
if let Some(why) = log_without_restriction(&value) {
return Err(refuse(error_problem(StatusCode::BAD_REQUEST, why)));
}
let (before, after) = (held(&current), held(&value));
if adds(&before, &after) {
require_tolerant_for(&state, &client, &agent)
.await
.map_err(refuse)?;
}
let revoking = revokes(&before, &after);
if revoking {
mirror(leaves, &agent, &before, &after)
.await
.map_err(|e| mirror_problem(&agent, &e, false))?;
}
if (current == value && !undecodable)
|| write(&store, &agent, &value, revision)
.await
.map_err(|e| queue_problem(&e))?
{
tracing::info!(%agent, ?value, "set agent permissions");
mirrored(&agent, &current, &value).await?;
if !revoking {
mirror(leaves, &agent, &before, &after)
.await
.map_err(|e| mirror_problem(&agent, &e, true))?;
}
return Ok(Json(view(value)));
}
}
@ -267,6 +314,16 @@ pub(crate) async fn put_agent_permissions(
pub(crate) async fn delete_agent_permissions(
State(state): State<AppState>,
Path(name): Path<String>,
) -> Result<StatusCode, ProblemDetails> {
delete_with(&observability_grant::Store, state, name).await
}
/// [`delete_agent_permissions`] against `leaves`. A clear that drops `logs` or
/// `metrics` writes the leaf first, as [`put_with`] does for a revoke.
async fn delete_with(
leaves: &impl GrantLeaves,
state: AppState,
name: String,
) -> Result<StatusCode, ProblemDetails> {
let refuse = |p| logged("delete", &name, p);
let agent = agent_name(&name).map_err(|r| refuse(refused(r)))?;
@ -284,6 +341,9 @@ pub(crate) async fn delete_agent_permissions(
internet_log: current.internet_log,
..AgentPermissions::default()
};
mirror(leaves, &agent, &held(&current), &held(&kept))
.await
.map_err(|e| mirror_problem(&agent, &e, false))?;
if kept == AgentPermissions::default() {
store
.delete(&agent)
@ -296,7 +356,6 @@ pub(crate) async fn delete_agent_permissions(
continue;
}
tracing::info!(%agent, "cleared agent permissions");
mirrored(&agent, &current, &kept).await?;
return Ok(StatusCode::NO_CONTENT);
}
Err(queue_problem(&anyhow::anyhow!(
@ -309,21 +368,20 @@ pub(crate) async fn delete_agent_permissions(
/// key left here would carry over to the next agent of that name.
///
/// Like the credential revoke it runs beside, a failure is logged and does
/// not stop the destroy; declaring the agent destroyed again re-runs it.
/// not stop the destroy; declaring the agent destroyed again re-runs it. The
/// mirrored grant goes first: if that write fails the record stays too, so the
/// two agree.
pub(crate) async fn clear_on_destroy(state: &AppState, hive: &str, agent: &str) {
clear_with(&observability_grant::Store, state, hive, agent).await;
}
async fn clear_with(leaves: &impl GrantLeaves, state: &AppState, hive: &str, agent: &str) {
let cleared = async {
let client = queue(state).map_err(|(_, why)| anyhow::anyhow!(why))?;
let store = agent_permissions::open_or_create(&client).await?;
let (current, _, _) = read_or_default::<AgentPermissions>(&store, agent).await?;
mirror(leaves, agent, &held(&current), &[]).await?;
store.delete(agent).await?;
// The mirrored grant goes with the record.
mirror(
&observability_grant::Store,
agent,
&current.permissions.unwrap_or_default(),
&[],
)
.await?;
anyhow::Ok(())
}
.await;
@ -525,6 +583,27 @@ pub(crate) fn spawn(client: async_nats::Client, wanted: Arc<WantedWriter>, hives
.await;
}
});
tokio::spawn({
let (store, wanted, hives) = (store.clone(), Arc::clone(&wanted), hives.clone());
async move {
// The seed's own retry has finished before the first pass, so
// a seed that waits on a hive does not race it.
loop {
match remirror(&store, &wanted, &hives).await {
Ok(0) => {}
Ok(n) => tracing::warn!(
n,
"agent permissions: corrected drifted log/metric grants"
),
Err(e) => tracing::warn!(
error = %format!("{e:#}"),
"agent permissions: re-mirroring the log/metric grants failed"
),
}
tokio::time::sleep(observability_grant::REMIRROR_EVERY).await;
}
}
});
let subject = import_subject_wildcard();
loop {
match client.subscribe(subject.clone()).await {
@ -551,6 +630,41 @@ pub(crate) fn spawn(client: async_nats::Client, wanted: Arc<WantedWriter>, hives
});
}
/// The grant each agent's record holds: every agent with a key, and every live
/// agent without one (the empty grant), so a leaf left for an agent whose
/// record is gone is found too.
async fn desired_grants(
store: &async_nats::jetstream::kv::Store,
wanted: &WantedWriter,
hives: &[String],
) -> anyhow::Result<BTreeMap<String, Grant>> {
let mut desired = BTreeMap::new();
let mut keys = store.keys().await?;
while let Some(key) = keys.next().await {
let key = key?;
let (record, _, _) = read_or_default::<AgentPermissions>(store, &key).await?;
desired.insert(key, grant_of(&held(&record)));
}
for agent in crate::subagent_term::live_agents(wanted, hives).await {
desired.entry(agent).or_default();
}
Ok(desired)
}
/// One pass of [`observability_grant::reconcile`] over every agent: the leaf
/// the gateways read becomes what the record says, so a grant whose mirror
/// write failed, or a revoke whose record write failed after its leaf, ends
/// consistent. `Ok` is the number of leaves corrected.
async fn remirror(
store: &async_nats::jetstream::kv::Store,
wanted: &WantedWriter,
hives: &[String],
) -> anyhow::Result<usize> {
let desired = desired_grants(store, wanted, hives).await?;
let leaves = observability_grant::Connected::login().await?;
observability_grant::reconcile(&leaves, &desired).await
}
async fn answer_import(
store: &async_nats::jetstream::kv::Store,
wanted: &WantedWriter,
@ -690,10 +804,12 @@ pub(crate) mod tests {
Permissions, delete_agent_permissions, get_agent_permissions, import_into,
put_agent_permissions, seed_into,
};
use crate::observability_grant::fake::Leaves;
use crate::test_nats::TestNats;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use hive_types::Permission;
use swarm_secret_client::observability::Grant;
use swarm_wire::agent_permissions::{Import, ImportReply};
fn stored(json: &str) -> swarm_wire::agent_permissions::AgentPermissions {
@ -1060,23 +1176,296 @@ pub(crate) mod tests {
.expect("control: no log or metric grant");
}
/// With the capability advertised the gate passes: the record is written,
/// and the mirror (which needs the secret store, absent here) is what
/// fails.
const TOLERANT_HIVE: &str = r#"{"capabilities":["tolerant-permissions"]}"#;
const LOGS_AND_METRICS: Grant = Grant {
logs: true,
metrics: true,
};
/// `state` with `agent_login` holding what each `(agent, capabilities)`
/// last reported.
fn with_reports(
state: crate::routes::AppState,
reports: &[(&str, &[&str])],
) -> crate::routes::AppState {
let reported = crate::agent_login::Reported::default();
for (agent, capabilities) in reports {
reported.record(
&swarm_queue_client::agent_state_subject(agent),
serde_json::json!({ "capabilities": capabilities })
.to_string()
.as_bytes(),
);
}
crate::routes::AppState {
agent_login: Some(std::sync::Arc::new(reported)),
..state
}
}
/// A state whose hive and whose atlas both advertise tolerant decoding.
async fn tolerant(client: &async_nats::Client) -> crate::routes::AppState {
let state = crate::agent_limits::tests::declared(client).await;
publish_snapshot(client, TOLERANT_HIVE).await;
with_reports(state, &[("atlas", &["tolerant-permissions"])])
}
async fn put_via(
leaves: &Leaves,
state: &crate::routes::AppState,
perms: &[Permission],
) -> Result<axum::Json<Permissions>, problem_details::ProblemDetails> {
super::put_with(
leaves,
state.clone(),
"atlas".to_owned(),
Ok(axum::Json(observing(perms))),
)
.await
}
async fn record(client: &async_nats::Client, perms: &str) {
swarm_queue_client::agent_permissions::open_or_create(client)
.await
.expect("bucket")
.put(
"atlas",
format!(r#"{{"permissions":{perms}}}"#).into_bytes().into(),
)
.await
.expect("record");
}
/// A tolerant hive is not enough: the agent's running harness has to have
/// reported the capability too. A harness that predates it, one that has
/// not reported yet and a hive that never published each refuse the grant.
#[tokio::test]
async fn an_advertising_hive_passes_the_gate() {
async fn the_agents_own_harness_must_report_the_capability() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
publish_snapshot(&client, r#"{"capabilities":["tolerant-permissions"]}"#).await;
publish_snapshot(&client, TOLERANT_HIVE).await;
let leaves = Leaves::default();
let problem = put(&state, "atlas", observing(&[Permission::Metrics]))
for (reports, why) in [
(&[][..], "no report yet"),
(&[("atlas", &[][..])][..], "a harness that lists none"),
(
&[("iris", &["tolerant-permissions"][..])][..],
"another agent",
),
] {
let state = with_reports(state.clone(), reports);
let err = put_via(&leaves, &state, &[Permission::Logs])
.await
.expect_err(why);
assert_eq!(err.status, Some(StatusCode::CONFLICT), "{why}");
assert!(err.detail.as_deref().unwrap_or_default().contains("atlas"));
assert_eq!(
published(&client, "atlas").await,
None,
"{why}: nothing written"
);
}
assert_eq!(leaves.get("atlas"), None);
// Control: with both reporting, the same request is accepted.
let state = with_reports(state, &[("atlas", &["tolerant-permissions"])]);
let _ = put_via(&leaves, &state, &[Permission::Logs])
.await
.expect_err("no secret store to mirror to");
assert_eq!(problem.status, Some(StatusCode::INTERNAL_SERVER_ERROR));
.expect("granted");
assert_eq!(leaves.get("atlas").map(|g| g.logs), Some(true));
}
/// Only a change that adds a grant needs the gate: an agent that holds
/// `logs` can have its other permissions changed while stopped.
#[tokio::test]
async fn keeping_a_grant_needs_no_report() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
let leaves = Leaves::default();
leaves.put("atlas", LOGS_AND_METRICS);
record(&client, r#"["logs","metrics"]"#).await;
let _ = put_via(
&leaves,
&state,
&[Permission::Logs, Permission::Metrics, Permission::WebTools],
)
.await
.expect("kept");
assert_eq!(leaves.get("atlas"), Some(LOGS_AND_METRICS));
}
/// A revoke writes the leaf first. With the store down it fails as an
/// error and changes nothing: the record and the leaf both still grant.
#[tokio::test]
async fn a_revoke_that_cannot_reach_the_store_changes_nothing() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
let leaves = Leaves::default();
leaves.put("atlas", LOGS_AND_METRICS);
record(&client, r#"["logs","metrics"]"#).await;
leaves.fail(true);
let err = put_via(&leaves, &state, &[Permission::Messaging])
.await
.expect_err("store down");
assert_eq!(err.status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["metrics"]}"#)
Some(r#"{"permissions":["logs","metrics"]}"#),
"the record still grants"
);
leaves.fail(false);
assert_eq!(
leaves.get("atlas"),
Some(LOGS_AND_METRICS),
"and so does the leaf"
);
// Control: with the store back the same request revokes both.
let _ = put_via(&leaves, &state, &[Permission::Messaging])
.await
.expect("revoked");
assert_eq!(leaves.get("atlas"), Some(Grant::default()));
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["messaging"]}"#)
);
}
/// A clear that drops a grant is ordered the same way.
#[tokio::test]
async fn a_clear_that_cannot_reach_the_store_changes_nothing() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
let leaves = Leaves::default();
leaves.put("atlas", LOGS_AND_METRICS);
record(&client, r#"["logs","metrics"]"#).await;
leaves.fail(true);
let err = super::delete_with(&leaves, state.clone(), "atlas".to_owned())
.await
.expect_err("store down");
assert_eq!(err.status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["logs","metrics"]}"#)
);
leaves.fail(false);
assert_eq!(leaves.get("atlas"), Some(LOGS_AND_METRICS));
super::delete_with(&leaves, state, "atlas".to_owned())
.await
.expect("cleared");
assert_eq!(leaves.get("atlas"), Some(Grant::default()));
assert_eq!(published(&client, "atlas").await, None);
}
/// Destroying an agent drops the grant leaf first too.
#[tokio::test]
async fn a_destroy_that_cannot_reach_the_store_keeps_both() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
let leaves = Leaves::default();
leaves.put("atlas", LOGS_AND_METRICS);
record(&client, r#"["logs"]"#).await;
leaves.fail(true);
super::clear_with(&leaves, &state, "pr1ma", "atlas").await;
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["logs"]}"#)
);
leaves.fail(false);
super::clear_with(&leaves, &state, "pr1ma", "atlas").await;
assert_eq!(published(&client, "atlas").await, None);
assert_eq!(leaves.get("atlas"), Some(Grant::default()));
}
/// A grant writes the record first. With the store down the request is an
/// error, the record grants and the leaf does not, so the gateway denies;
/// the periodic pass then writes the leaf.
#[tokio::test]
async fn a_grant_that_cannot_reach_the_store_leaves_the_gateway_denying_until_the_pass() {
let nats = TestNats::start();
let client = nats.client().await;
let state = tolerant(&client).await;
let leaves = Leaves::default();
leaves.fail(true);
let err = put_via(&leaves, &state, &[Permission::Logs, Permission::Metrics])
.await
.expect_err("store down");
assert_eq!(err.status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(
published(&client, "atlas").await.as_deref(),
Some(r#"{"permissions":["logs","metrics"]}"#),
"the record grants"
);
leaves.fail(false);
assert_eq!(leaves.get("atlas"), None, "the leaf does not");
let store = swarm_queue_client::agent_permissions::open_or_create(&client)
.await
.expect("bucket");
let wanted = state.wanted.as_ref().expect("wanted");
let desired = super::desired_grants(&store, wanted, &["pr1ma".to_owned()])
.await
.expect("desired");
assert_eq!(
crate::observability_grant::reconcile(&leaves, &desired)
.await
.expect("pass"),
1
);
assert_eq!(leaves.get("atlas"), Some(LOGS_AND_METRICS));
}
/// The pass reads the records, not the leaves: it revokes a leaf whose
/// agent has no record or an empty one, and names every live agent.
#[tokio::test]
async fn the_pass_makes_leaves_follow_records_in_both_directions() {
let nats = TestNats::start();
let client = nats.client().await;
let state = crate::agent_limits::tests::declared(&client).await;
let store = swarm_queue_client::agent_permissions::open_or_create(&client)
.await
.expect("bucket");
store
.put("iris", br#"{"permissions":["metrics"]}"#.to_vec().into())
.await
.expect("iris");
let leaves = Leaves::default();
leaves.put("atlas", LOGS_AND_METRICS); // live, no record: revoke
leaves.put("iris", Grant::default()); // record grants metrics: grant
let wanted = state.wanted.as_ref().expect("wanted");
let desired = super::desired_grants(&store, wanted, &["pr1ma".to_owned()])
.await
.expect("desired");
assert_eq!(
desired.keys().map(String::as_str).collect::<Vec<_>>(),
["atlas", "iris"]
);
assert_eq!(
crate::observability_grant::reconcile(&leaves, &desired)
.await
.expect("pass"),
2
);
assert_eq!(leaves.get("atlas"), Some(Grant::default()));
assert_eq!(
leaves.get("iris"),
Some(Grant {
logs: false,
metrics: true
})
);
}