order grant mirror writes by direction, re-mirror every five minutes, gate on the agent's own harness
This commit is contained in:
atlas
parent
8a60f52afa
commit
329d556ab9
1 changed files with 793 additions and 118 deletions
|
|
@ -1,23 +1,38 @@
|
|||
//! The `logs` and `metrics` permissions beyond the queue's permissions bucket:
|
||||
//! which hives may be granted them, and the mirror of the grant that the store
|
||||
//! which agents may be granted them, and the mirror of the grant that the store
|
||||
//! gateways read.
|
||||
//!
|
||||
//! A hive that cannot skip a permission name it does not know would fail to
|
||||
//! decode a record that holds one, so the controller grants them only on a hive
|
||||
//! whose status snapshot advertises `swarm_wire::hive_status::TOLERANT_PERMISSIONS`.
|
||||
//! `swarm-logs-auth` has no queue access, so each grant is also written to
|
||||
//! An agent that cannot skip a permission name it does not know would drop to
|
||||
//! no permissions on reading a record that holds one, so the controller grants
|
||||
//! them only when the agent's hive status snapshot and its running harness's
|
||||
//! turn-state header both advertise
|
||||
//! `swarm_wire::hive_status::TOLERANT_PERMISSIONS`. `swarm-logs-auth` has no
|
||||
//! queue access, so each grant is also written to
|
||||
//! `swarm_secret_client::observability::grant_path`; a leaf that is absent is no
|
||||
//! grant.
|
||||
//!
|
||||
//! **The leaf is what the gateway enforces, the record is what the operator
|
||||
//! sees**, so a change is ordered to fail towards denying: a revoke writes the
|
||||
//! leaf first and the record second, a grant the record first and the leaf
|
||||
//! second. [`reconcile`] makes every leaf equal its record's grant, every
|
||||
//! [`REMIRROR_EVERY`], for the writes that failed half way.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use hive_types::Permission;
|
||||
use swarm_secret_client::SecretStore;
|
||||
use swarm_secret_client::observability::{Grant, grant_path};
|
||||
use swarm_wire::hive_status::{Capabilities, TOLERANT_PERMISSIONS};
|
||||
|
||||
use crate::wanted::WantedWriter;
|
||||
|
||||
/// How often [`reconcile`] runs, matching the controller's other periodic
|
||||
/// passes over the swarm's agents (`agent_renewal`).
|
||||
pub(crate) const REMIRROR_EVERY: Duration = Duration::from_mins(5);
|
||||
|
||||
/// The grant `perms` hold.
|
||||
pub(crate) fn grant_of(perms: &[Permission]) -> Grant {
|
||||
Grant {
|
||||
|
|
@ -26,20 +41,65 @@ pub(crate) fn grant_of(perms: &[Permission]) -> Grant {
|
|||
}
|
||||
}
|
||||
|
||||
/// Where a mirrored grant is written. A test stands in for the store.
|
||||
pub(crate) trait GrantSink {
|
||||
/// Whether `after` holds a grant `before` did not.
|
||||
pub(crate) fn adds(before: &[Permission], after: &[Permission]) -> bool {
|
||||
let (b, a) = (grant_of(before), grant_of(after));
|
||||
(a.logs && !b.logs) || (a.metrics && !b.metrics)
|
||||
}
|
||||
|
||||
/// Whether `after` drops a grant `before` held.
|
||||
pub(crate) fn revokes(before: &[Permission], after: &[Permission]) -> bool {
|
||||
adds(after, before)
|
||||
}
|
||||
|
||||
/// Where a mirrored grant is read and written. A test stands in for the store.
|
||||
pub(crate) trait GrantLeaves {
|
||||
/// The leaf, `None` when nothing is stored there.
|
||||
fn read(&self, agent: &str) -> impl Future<Output = Result<Option<Grant>>> + Send;
|
||||
fn write(&self, agent: &str, grant: Grant) -> impl Future<Output = Result<()>> + Send;
|
||||
}
|
||||
|
||||
/// The swarm secret store, under the controller's own identity.
|
||||
/// The swarm secret store, under the controller's own identity, logging in
|
||||
/// for each call.
|
||||
pub(crate) struct Store;
|
||||
|
||||
impl GrantSink for Store {
|
||||
impl GrantLeaves for Store {
|
||||
async fn read(&self, agent: &str) -> Result<Option<Grant>> {
|
||||
Connected(connect().await?).read(agent).await
|
||||
}
|
||||
|
||||
async fn write(&self, agent: &str, grant: Grant) -> Result<()> {
|
||||
Connected(connect().await?).write(agent, grant).await
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect() -> Result<SecretStore> {
|
||||
crate::store::connect()
|
||||
.await
|
||||
.context("logging in to the swarm secret store")
|
||||
}
|
||||
|
||||
/// One login, for a pass that touches many leaves.
|
||||
pub(crate) struct Connected(pub(crate) SecretStore);
|
||||
|
||||
impl Connected {
|
||||
pub(crate) async fn login() -> Result<Self> {
|
||||
Ok(Self(connect().await?))
|
||||
}
|
||||
}
|
||||
|
||||
impl GrantLeaves for Connected {
|
||||
async fn read(&self, agent: &str) -> Result<Option<Grant>> {
|
||||
let path = grant_path(agent)?;
|
||||
self.0
|
||||
.read_optional(&path)
|
||||
.await
|
||||
.with_context(|| format!("reading {path}"))
|
||||
}
|
||||
|
||||
async fn write(&self, agent: &str, grant: Grant) -> Result<()> {
|
||||
let path = grant_path(agent)?;
|
||||
crate::store::connect()
|
||||
.await
|
||||
.context("logging in to the swarm secret store")?
|
||||
self.0
|
||||
.write(&path, &grant)
|
||||
.await
|
||||
.with_context(|| format!("writing {path}"))
|
||||
|
|
@ -51,9 +111,9 @@ impl GrantSink for Store {
|
|||
/// leaves the leaf absent.
|
||||
///
|
||||
/// # Errors
|
||||
/// The write failed; the responder then keeps the grant it last read, or none.
|
||||
/// The write failed; the gateway then keeps the grant it last read, or none.
|
||||
pub(crate) async fn mirror(
|
||||
sink: &impl GrantSink,
|
||||
leaves: &impl GrantLeaves,
|
||||
agent: &str,
|
||||
before: &[Permission],
|
||||
after: &[Permission],
|
||||
|
|
@ -61,7 +121,52 @@ pub(crate) async fn mirror(
|
|||
if grant_of(before) == Grant::default() && grant_of(after) == Grant::default() {
|
||||
return Ok(());
|
||||
}
|
||||
sink.write(agent, grant_of(after)).await
|
||||
leaves.write(agent, grant_of(after)).await
|
||||
}
|
||||
|
||||
/// Make each agent's leaf equal `desired`, the grant its record holds; an
|
||||
/// absent leaf equals the empty grant. Every correction is logged at `warn`.
|
||||
/// All agents are tried; `Ok` is the number corrected.
|
||||
///
|
||||
/// # Errors
|
||||
/// How many agents could not be read or written.
|
||||
pub(crate) async fn reconcile(
|
||||
leaves: &impl GrantLeaves,
|
||||
desired: &BTreeMap<String, Grant>,
|
||||
) -> Result<usize> {
|
||||
let (mut corrected, mut failed) = (0, 0);
|
||||
for (agent, want) in desired {
|
||||
let have = match leaves.read(agent).await {
|
||||
Ok(leaf) => leaf.unwrap_or_default(),
|
||||
Err(e) => {
|
||||
tracing::warn!(agent, error = %format!("{e:#}"), "observability grant: reading the leaf failed");
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if have == *want {
|
||||
continue;
|
||||
}
|
||||
match leaves.write(agent, *want).await {
|
||||
Ok(()) => {
|
||||
tracing::warn!(
|
||||
agent,
|
||||
?have,
|
||||
?want,
|
||||
"observability grant: leaf differed from the record; corrected"
|
||||
);
|
||||
corrected += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(agent, error = %format!("{e:#}"), "observability grant: correcting the leaf failed");
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
anyhow::bail!("{failed} agent(s) could not be reconciled");
|
||||
}
|
||||
Ok(corrected)
|
||||
}
|
||||
|
||||
/// `Ok` when `hive`'s snapshot advertises tolerant decoding.
|
||||
|
|
@ -128,22 +233,34 @@ pub(crate) async fn hive_is_tolerant(
|
|||
Ok(check_capability(&hive, Some(&snapshot)).is_ok())
|
||||
}
|
||||
|
||||
/// `Ok` when the hive running `agent` advertises tolerant decoding, so the
|
||||
/// agent's record may name `logs` or `metrics`.
|
||||
/// `Ok` when `agent` may be granted `logs` or `metrics`: its hive advertises
|
||||
/// tolerant decoding and so does the harness it is running (its latest
|
||||
/// turn-state header, which an agent that is stopped, not yet restarted on a
|
||||
/// tolerant build, or not reporting yet does not send).
|
||||
///
|
||||
/// # Errors
|
||||
/// The reason: no hive declares the agent, the hive's snapshot is missing or
|
||||
/// lacks the capability, or the queue could not be read.
|
||||
pub(crate) async fn require_tolerant_hive(
|
||||
/// lacks the capability, the harness has not reported it, or the queue could
|
||||
/// not be read.
|
||||
pub(crate) async fn require_tolerant_agent(
|
||||
client: &async_nats::Client,
|
||||
wanted: &WantedWriter,
|
||||
hives: &[String],
|
||||
reported: Option<&crate::agent_login::Reported>,
|
||||
agent: &str,
|
||||
) -> Result<(), String> {
|
||||
let hive = hive_of(wanted, hives, agent)
|
||||
.await
|
||||
.ok_or_else(|| format!("no hive declares agent {agent}"))?;
|
||||
require_tolerant(client, &hive).await
|
||||
require_tolerant(client, &hive).await?;
|
||||
if reported.is_some_and(|r| r.advertises(agent, TOLERANT_PERMISSIONS)) {
|
||||
return Ok(());
|
||||
}
|
||||
Err(format!(
|
||||
"agent {agent}'s harness has not reported {TOLERANT_PERMISSIONS}: it must be running \
|
||||
a build that decodes the new permissions (update it, then wait a minute for its next \
|
||||
status)"
|
||||
))
|
||||
}
|
||||
|
||||
/// `Ok` when `hive`'s last status snapshot advertises tolerant decoding.
|
||||
|
|
@ -160,49 +277,89 @@ pub(crate) async fn require_tolerant(
|
|||
check_capability(hive, snapshot.as_ref())
|
||||
}
|
||||
|
||||
/// A stand-in for the store's leaves, shared by the tests that mirror.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
pub(crate) mod fake {
|
||||
use super::{GrantLeaves, Result};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use swarm_secret_client::observability::Grant;
|
||||
|
||||
/// Leaves by agent. While `failing`, every read and write is an error.
|
||||
#[derive(Default)]
|
||||
struct Leaves(Mutex<Vec<(String, Grant)>>);
|
||||
pub(crate) struct Leaves {
|
||||
pub(crate) held: Mutex<BTreeMap<String, Grant>>,
|
||||
pub(crate) failing: AtomicBool,
|
||||
}
|
||||
|
||||
impl GrantSink for &Leaves {
|
||||
async fn write(&self, agent: &str, grant: Grant) -> Result<()> {
|
||||
self.0.lock().unwrap().push((agent.to_owned(), grant));
|
||||
Ok(())
|
||||
impl Leaves {
|
||||
pub(crate) fn fail(&self, on: bool) {
|
||||
self.failing.store(on, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, agent: &str) -> Option<Grant> {
|
||||
self.held.lock().unwrap().get(agent).copied()
|
||||
}
|
||||
|
||||
pub(crate) fn put(&self, agent: &str, grant: Grant) {
|
||||
self.held.lock().unwrap().insert(agent.to_owned(), grant);
|
||||
}
|
||||
}
|
||||
|
||||
struct Down;
|
||||
|
||||
impl GrantSink for Down {
|
||||
async fn write(&self, _: &str, _: Grant) -> Result<()> {
|
||||
anyhow::bail!("store unreachable")
|
||||
impl GrantLeaves for Leaves {
|
||||
async fn read(&self, agent: &str) -> Result<Option<Grant>> {
|
||||
anyhow::ensure!(!self.failing.load(Ordering::SeqCst), "store unreachable");
|
||||
Ok(self.get(agent))
|
||||
}
|
||||
|
||||
async fn write(&self, agent: &str, grant: Grant) -> Result<()> {
|
||||
anyhow::ensure!(!self.failing.load(Ordering::SeqCst), "store unreachable");
|
||||
self.put(agent, grant);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::fake::Leaves;
|
||||
use super::*;
|
||||
|
||||
const YES: Grant = Grant {
|
||||
logs: true,
|
||||
metrics: true,
|
||||
};
|
||||
const LOGS: Grant = Grant {
|
||||
logs: true,
|
||||
metrics: false,
|
||||
};
|
||||
const BOTH: [Permission; 3] = [Permission::Messaging, Permission::Logs, Permission::Metrics];
|
||||
const PLAIN: [Permission; 1] = [Permission::Messaging];
|
||||
|
||||
#[test]
|
||||
fn adds_and_revokes_are_the_two_directions() {
|
||||
assert!(adds(&PLAIN, &BOTH) && !revokes(&PLAIN, &BOTH));
|
||||
assert!(revokes(&BOTH, &PLAIN) && !adds(&BOTH, &PLAIN));
|
||||
assert!(
|
||||
adds(&[Permission::Logs], &[Permission::Metrics]),
|
||||
"swap adds"
|
||||
);
|
||||
assert!(
|
||||
revokes(&[Permission::Logs], &[Permission::Metrics]),
|
||||
"and revokes"
|
||||
);
|
||||
assert!(!adds(&BOTH, &BOTH) && !revokes(&BOTH, &BOTH));
|
||||
assert!(!adds(&PLAIN, &PLAIN) && !revokes(&PLAIN, &PLAIN));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_grant_and_its_revocation_are_written() {
|
||||
let leaves = Leaves::default();
|
||||
let both = [Permission::Messaging, Permission::Logs, Permission::Metrics];
|
||||
mirror(&&leaves, "atlas", &[], &both).await.unwrap();
|
||||
mirror(&&leaves, "atlas", &both, &[Permission::Messaging])
|
||||
.await
|
||||
.unwrap();
|
||||
let written = leaves.0.lock().unwrap().clone();
|
||||
let yes = Grant {
|
||||
logs: true,
|
||||
metrics: true,
|
||||
};
|
||||
assert_eq!(
|
||||
written,
|
||||
[
|
||||
("atlas".to_owned(), yes),
|
||||
("atlas".to_owned(), Grant::default())
|
||||
]
|
||||
);
|
||||
mirror(&leaves, "atlas", &[], &BOTH).await.unwrap();
|
||||
assert_eq!(leaves.get("atlas"), Some(YES));
|
||||
mirror(&leaves, "atlas", &BOTH, &PLAIN).await.unwrap();
|
||||
assert_eq!(leaves.get("atlas"), Some(Grant::default()));
|
||||
}
|
||||
|
||||
/// An agent that never held either, before or after, gets no leaf; one
|
||||
|
|
@ -210,23 +367,70 @@ mod tests {
|
|||
#[tokio::test]
|
||||
async fn only_the_permissions_held_are_mirrored() {
|
||||
let leaves = Leaves::default();
|
||||
let plain = [Permission::Messaging];
|
||||
mirror(&&leaves, "atlas", &plain, &plain).await.unwrap();
|
||||
assert!(leaves.0.lock().unwrap().is_empty());
|
||||
mirror(&&leaves, "atlas", &plain, &[Permission::Metrics])
|
||||
mirror(&leaves, "atlas", &PLAIN, &PLAIN).await.unwrap();
|
||||
assert_eq!(leaves.get("atlas"), None);
|
||||
mirror(&leaves, "atlas", &PLAIN, &[Permission::Logs])
|
||||
.await
|
||||
.unwrap();
|
||||
let written = leaves.0.lock().unwrap().clone();
|
||||
assert_eq!(written.len(), 1);
|
||||
assert_eq!((written[0].1.logs, written[0].1.metrics), (false, true));
|
||||
assert_eq!(leaves.get("atlas"), Some(LOGS));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_write_is_an_error() {
|
||||
let held = [Permission::Logs];
|
||||
assert!(mirror(&Down, "atlas", &[], &held).await.is_err());
|
||||
let leaves = Leaves::default();
|
||||
leaves.fail(true);
|
||||
assert!(mirror(&leaves, "atlas", &[], &BOTH).await.is_err());
|
||||
// Control: nothing to write is not a failure, whatever the store does.
|
||||
assert!(mirror(&Down, "atlas", &[], &[]).await.is_ok());
|
||||
assert!(mirror(&leaves, "atlas", &[], &[]).await.is_ok());
|
||||
}
|
||||
|
||||
/// The pass corrects a leaf that is granted where the record is not, one
|
||||
/// that is missing or short where the record grants, and leaves the
|
||||
/// agreeing ones (and an absent leaf for an empty grant) alone.
|
||||
#[tokio::test]
|
||||
async fn the_pass_corrects_drift_in_both_directions() {
|
||||
let leaves = Leaves::default();
|
||||
leaves.put("stale", YES);
|
||||
leaves.put("short", LOGS);
|
||||
leaves.put("same", YES);
|
||||
let desired = BTreeMap::from([
|
||||
("stale".to_owned(), Grant::default()),
|
||||
("short".to_owned(), YES),
|
||||
("missing".to_owned(), YES),
|
||||
("same".to_owned(), YES),
|
||||
("none".to_owned(), Grant::default()),
|
||||
]);
|
||||
assert_eq!(reconcile(&leaves, &desired).await.unwrap(), 3);
|
||||
assert_eq!(
|
||||
leaves.get("stale"),
|
||||
Some(Grant::default()),
|
||||
"revoke repaired"
|
||||
);
|
||||
assert_eq!(leaves.get("short"), Some(YES));
|
||||
assert_eq!(leaves.get("missing"), Some(YES), "grant repaired");
|
||||
assert_eq!(leaves.get("same"), Some(YES));
|
||||
assert_eq!(leaves.get("none"), None, "no leaf made for an empty grant");
|
||||
assert_eq!(reconcile(&leaves, &desired).await.unwrap(), 0, "converged");
|
||||
}
|
||||
|
||||
/// A store that is down is an error and changes nothing; the next pass,
|
||||
/// once it is back, fixes it.
|
||||
#[tokio::test]
|
||||
async fn a_failed_pass_changes_nothing_and_the_next_one_fixes_it() {
|
||||
let leaves = Leaves::default();
|
||||
leaves.put("stale", YES);
|
||||
let desired = BTreeMap::from([
|
||||
("stale".to_owned(), Grant::default()),
|
||||
("missing".to_owned(), YES),
|
||||
]);
|
||||
leaves.fail(true);
|
||||
assert!(reconcile(&leaves, &desired).await.is_err());
|
||||
leaves.fail(false);
|
||||
assert_eq!(leaves.get("stale"), Some(YES));
|
||||
assert_eq!(leaves.get("missing"), None);
|
||||
assert_eq!(reconcile(&leaves, &desired).await.unwrap(), 2);
|
||||
assert_eq!(leaves.get("stale"), Some(Grant::default()));
|
||||
assert_eq!(leaves.get("missing"), Some(YES));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue