Watch
0
0
Fork
You've already forked hyperhive
0
hyperhive/swarm-controller/src/archive_access.rs
atlas 91f36f39d2 swarm-archive: grant only once an archive is up; keep logins across a gap in placements
The archive host creates a ready marker in the store and swarm-controller
writes no credentials before it exists, so a swarm without an archive, and a
mixed-version swarm, leave every hive backup a logged no-op. A hive with no
declaration no longer counts as placing nothing: its copies and its agents'
user lines stay. The restic password is pinned by tests across a revoke, a
re-placement and a move.
2026-11-10 08:18:45 +01:00

682 lines
25 KiB
Rust

//! Each placed agent's archive credentials, and their revocation.
//!
//! The archive (`swarm-archive.nix`) keeps one restic repository per agent and
//! lets a user reach only its own. Every [`RECONCILE_INTERVAL`] this makes the
//! store hold, per `(hive, agent)` that [`crate::jobq::placements`] reports:
//!
//! - the REST password and the restic password at
//! [`archive::grant_path`], which that hive's store identity reads;
//! - the htpasswd line, a hash of the REST password, at [`archive::user_path`],
//! which the archive host renders its users file from;
//! - the restic password at [`archive::restic_path`], created once and never
//! replaced: it opens the agent's repository wherever the agent runs, and the
//! archive host prunes with it.
//!
//! A pass is level-triggered and idempotent. A move to another hive mints a new
//! REST password, which replaces the htpasswd line, so the old hive's password
//! stops working once the archive host re-renders; the old hive's copy is
//! deleted in the same pass. An agent no hive places loses its htpasswd line and
//! every hive's copy. Its restic password stays, so what was backed up can still
//! be read.
//!
//! An agent that two hives place is left as it is: either grant would replace
//! the other's every pass.
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result};
use base64::Engine as _;
use sha1::{Digest as _, Sha1};
use swarm_secret_client::SecretStore;
use swarm_secret_client::archive::{self, HTPASSWD, PASSWORD, REST_PASSWORD, RESTIC_PASSWORD};
use crate::wanted::WantedWriter;
/// How often [`spawn`] reconciles.
const RECONCILE_INTERVAL: Duration = Duration::from_mins(5);
/// A pass that has not finished by now is given up on; the next tick retries.
const PASS_DEADLINE: Duration = Duration::from_mins(2);
/// An object in the store: field name to value.
type Fields = BTreeMap<String, String>;
/// The store calls a pass makes, so a test can stand in for the store.
pub(crate) trait ArchiveStore {
/// The object at `path`; `None` when nothing is stored there.
async fn read(&self, path: &str) -> Result<Option<Fields>>;
/// Store `fields` at `path`, replacing what is there.
async fn write(&self, path: &str, fields: &Fields) -> Result<()>;
/// Store `fields` at `path` only while nothing is there.
async fn create(&self, path: &str, fields: &Fields) -> Result<()>;
/// The names stored directly under `path`.
async fn list(&self, path: &str) -> Result<Vec<String>>;
/// Remove `path` with every version of it; nothing there is not an error.
async fn delete(&self, path: &str) -> Result<()>;
}
impl ArchiveStore for SecretStore {
async fn read(&self, path: &str) -> Result<Option<Fields>> {
Ok(self.read_optional(path).await?)
}
async fn write(&self, path: &str, fields: &Fields) -> Result<()> {
Ok(SecretStore::write(self, path, fields).await?)
}
async fn create(&self, path: &str, fields: &Fields) -> Result<()> {
Ok(SecretStore::create(self, path, fields).await?)
}
async fn list(&self, path: &str) -> Result<Vec<String>> {
Ok(SecretStore::list(self, path).await?)
}
async fn delete(&self, path: &str) -> Result<()> {
Ok(self.delete_all_versions(path).await?)
}
}
/// What one pass did.
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct Done {
/// Agents whose credentials were written or replaced.
pub granted: usize,
/// Credentials removed: a hive's copy or an htpasswd line.
pub revoked: usize,
/// Agents or hives whose credentials could not all be checked or written.
pub failed: usize,
}
fn fields(pairs: &[(&str, &str)]) -> Fields {
pairs
.iter()
.map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))
.collect()
}
/// The non-empty field `name` of `object`.
fn field<'a>(object: Option<&'a Fields>, name: &str) -> Option<&'a str> {
object
.and_then(|o| o.get(name))
.map(String::as_str)
.filter(|v| !v.is_empty())
}
/// The htpasswd line that accepts `password` for `agent`: apache's `{SHA}`
/// form, which the password's 256 random bits make as strong as a slower hash.
fn htpasswd_line(agent: &str, password: &str) -> String {
let digest = Sha1::digest(password.as_bytes());
format!(
"{agent}:{{SHA}}{}",
base64::engine::general_purpose::STANDARD.encode(digest)
)
}
/// `agent`'s restic password: the stored one, else a new one created at
/// [`archive::restic_path`]. A create that fails is read back, in case another
/// writer's landed first.
async fn ensure_restic(store: &impl ArchiveStore, agent: &str) -> Result<String> {
let path = archive::restic_path(agent)?;
let stored = store
.read(&path)
.await
.with_context(|| format!("reading {path}"))?;
if let Some(password) = field(stored.as_ref(), PASSWORD) {
return Ok(password.to_owned());
}
let minted = crate::agent_identity::generate_queue_secret()?;
if let Err(e) = store.create(&path, &fields(&[(PASSWORD, &minted)])).await {
let winner = store
.read(&path)
.await
.with_context(|| format!("reading {path} back after a failed create"))?;
return field(winner.as_ref(), PASSWORD)
.map(str::to_owned)
.ok_or_else(|| e.context(format!("creating {path}")));
}
Ok(minted)
}
/// Bring `agent`'s credentials on `hive` and at the archive up to date; whether
/// anything was written.
async fn grant(store: &impl ArchiveStore, hive: &str, agent: &str) -> Result<bool> {
let restic = ensure_restic(store, agent).await?;
let grant_path = archive::grant_path(hive, agent)?;
let held = store
.read(&grant_path)
.await
.with_context(|| format!("reading {grant_path}"))?;
let mut written = false;
let rest = match field(held.as_ref(), REST_PASSWORD) {
Some(rest) if field(held.as_ref(), RESTIC_PASSWORD) == Some(restic.as_str()) => {
rest.to_owned()
}
_ => {
let rest = crate::agent_identity::generate_queue_secret()?;
store
.write(
&grant_path,
&fields(&[(REST_PASSWORD, &rest), (RESTIC_PASSWORD, &restic)]),
)
.await
.with_context(|| format!("writing {grant_path}"))?;
written = true;
rest
}
};
let user_path = archive::user_path(agent)?;
let line = htpasswd_line(agent, &rest);
let user = store
.read(&user_path)
.await
.with_context(|| format!("reading {user_path}"))?;
if field(user.as_ref(), HTPASSWD) != Some(line.as_str()) {
store
.write(&user_path, &fields(&[(HTPASSWD, &line)]))
.await
.with_context(|| format!("writing {user_path}"))?;
written = true;
}
Ok(written)
}
/// The entries under `prefix` that are single leaves.
async fn leaves(store: &impl ArchiveStore, prefix: &str) -> Result<Vec<String>> {
Ok(store
.list(prefix)
.await
.with_context(|| format!("listing {prefix}"))?
.into_iter()
.filter(|name| !name.ends_with('/'))
.collect())
}
/// One pass: grant every agent in `placed`, then revoke what no placement
/// justifies. `hives` is every hive of the swarm, whose copies are checked.
///
/// Nothing happens until the archive has announced itself at
/// [`archive::ready_path`]: a swarm without one gets no credentials. A hive in
/// `undeclared` has no declaration to read, which does not show it places
/// nothing, so its copies, and the user lines of the agents they name, stay.
///
/// # Errors
/// Never: a failure is counted in [`Done::failed`] and logged, and the rest of
/// the pass still runs.
pub(crate) async fn reconcile(
store: &impl ArchiveStore,
placed: &BTreeSet<(String, String)>,
undeclared: &BTreeSet<String>,
hives: &[String],
) -> Done {
let mut done = Done::default();
match store.read(&archive::ready_path().unwrap_or_default()).await {
Ok(Some(_)) => {}
Ok(None) => return done,
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "archive: reading whether the archive is up failed");
done.failed += 1;
return done;
}
}
let mut holders: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
for (hive, agent) in placed {
holders.entry(agent).or_default().push(hive);
}
let mut ambiguous = BTreeSet::new();
let mut unconfirmed = BTreeSet::new();
for (agent, on) in &holders {
if let [hive] = on.as_slice() {
match grant(store, hive, agent).await {
Ok(true) => done.granted += 1,
Ok(false) => {}
Err(e) => {
tracing::warn!(hive, agent, error = %format!("{e:#}"), "archive: granting failed");
done.failed += 1;
}
}
} else {
tracing::warn!(agent, hives = ?on, "archive: placed on more than one hive; credentials left as they are");
ambiguous.insert(*agent);
}
}
for hive in hives {
let copies = match archive::grants_prefix(hive) {
Ok(prefix) => leaves(store, &prefix).await,
Err(e) => Err(e.into()),
};
let copies = match copies {
Ok(copies) => copies,
Err(e) => {
tracing::warn!(hive, error = %format!("{e:#}"), "archive: listing the hive's copies failed");
done.failed += 1;
continue;
}
};
if undeclared.contains(hive) {
unconfirmed.extend(copies);
continue;
}
for agent in copies {
let keep = ambiguous.contains(agent.as_str())
|| placed.contains(&(hive.clone(), agent.clone()));
if !keep {
revoke(store, archive::grant_path(hive, &agent), &mut done).await;
}
}
}
match leaves(store, &archive::users_prefix().unwrap_or_default()).await {
Ok(users) => {
for agent in users {
if !holders.contains_key(agent.as_str()) && !unconfirmed.contains(&agent) {
revoke(store, archive::user_path(&agent), &mut done).await;
}
}
}
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "archive: listing the users failed");
done.failed += 1;
}
}
done
}
/// Delete the credential at `path`, counting the outcome in `done`.
async fn revoke(
store: &impl ArchiveStore,
path: Result<String, swarm_secret_client::Error>,
done: &mut Done,
) {
let path = match path {
Ok(path) => path,
Err(e) => {
tracing::warn!(error = %e, "archive: a stored name is not a valid path segment");
done.failed += 1;
return;
}
};
match store.delete(&path).await {
Ok(()) => {
tracing::info!(%path, "archive: credential revoked");
done.revoked += 1;
}
Err(e) => {
tracing::warn!(%path, error = %format!("{e:#}"), "archive: revoking failed");
done.failed += 1;
}
}
}
/// Reconcile now and every [`RECONCILE_INTERVAL`] after. A pass that fails is
/// logged and retried on the next tick; it never stops the daemon. Nothing is
/// revoked on a pass that could not read where the hives place agents.
pub fn spawn(wanted: Arc<WantedWriter>, hives: Vec<String>) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
loop {
ticker.tick().await;
let pass = async {
let (placed, undeclared) =
crate::jobq::placements_with_gaps(&wanted, &hives).await?;
let store = crate::store::connect()
.await
.context("logging in to the swarm secret store")?;
anyhow::Ok(reconcile(&store, &placed, &undeclared, &hives).await)
};
match tokio::time::timeout(PASS_DEADLINE, pass).await {
Ok(Ok(done)) if done != Done::default() => tracing::info!(
granted = done.granted,
revoked = done.revoked,
failed = done.failed,
"archive: credentials reconciled"
),
Ok(Ok(_)) => tracing::debug!("archive: every credential is current"),
Ok(Err(e)) => tracing::warn!(
error = %format!("{e:#}"),
retry_in_s = RECONCILE_INTERVAL.as_secs(),
"archive: pass failed; retrying next tick"
),
Err(_) => tracing::warn!(
deadline_s = PASS_DEADLINE.as_secs(),
"archive: pass timed out; retrying next tick"
),
}
}
});
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
/// An in-memory store. `create` refuses a held path, `list` answers like
/// the store's: the names directly under a prefix.
#[derive(Default)]
struct Memory {
kv: Mutex<BTreeMap<String, Fields>>,
/// Paths under this prefix refuse every write and delete.
refuse: Option<&'static str>,
}
impl Memory {
/// A store whose archive has announced itself.
fn up() -> Self {
let store = Self::default();
store.kv.lock().unwrap().insert(
archive::ready_path().unwrap(),
fields(&[(archive::READY, "1")]),
);
store
}
fn get(&self, path: &str) -> Option<Fields> {
self.kv.lock().unwrap().get(path).cloned()
}
fn value(&self, path: &str, name: &str) -> Option<String> {
self.get(path).and_then(|f| f.get(name).cloned())
}
fn refused(&self, path: &str) -> bool {
self.refuse.is_some_and(|p| path.starts_with(p))
}
}
impl ArchiveStore for Memory {
async fn read(&self, path: &str) -> Result<Option<Fields>> {
Ok(self.get(path))
}
async fn write(&self, path: &str, fields: &Fields) -> Result<()> {
if self.refused(path) {
anyhow::bail!("denied");
}
self.kv
.lock()
.unwrap()
.insert(path.to_owned(), fields.clone());
Ok(())
}
async fn create(&self, path: &str, fields: &Fields) -> Result<()> {
if self.get(path).is_some() {
anyhow::bail!("exists");
}
self.write(path, fields).await
}
async fn list(&self, path: &str) -> Result<Vec<String>> {
let prefix = format!("{path}/");
Ok(self
.kv
.lock()
.unwrap()
.keys()
.filter_map(|k| k.strip_prefix(&prefix))
.map(|rest| match rest.split_once('/') {
Some((dir, _)) => format!("{dir}/"),
None => rest.to_owned(),
})
.collect())
}
async fn delete(&self, path: &str) -> Result<()> {
if self.refused(path) {
anyhow::bail!("denied");
}
self.kv.lock().unwrap().remove(path);
Ok(())
}
}
fn placed(pairs: &[(&str, &str)]) -> BTreeSet<(String, String)> {
pairs
.iter()
.map(|(h, a)| ((*h).to_owned(), (*a).to_owned()))
.collect()
}
fn none() -> BTreeSet<String> {
BTreeSet::new()
}
fn hives() -> Vec<String> {
vec!["alpha".to_owned(), "beta".to_owned()]
}
const ALPHA_ATLAS: &str = "swarm/hives/alpha/archive/atlas";
const BETA_ATLAS: &str = "swarm/hives/beta/archive/atlas";
const USER_ATLAS: &str = "swarm/services/archive/users/atlas";
const RESTIC_ATLAS: &str = "swarm/services/archive/restic/atlas";
#[test]
fn the_htpasswd_line_is_apaches_sha_form() {
// `htpasswd -nbs atlas secret`
assert_eq!(
htpasswd_line("atlas", "secret"),
"atlas:{SHA}5en6G6MezRroT3XKqkdPOmY/BfQ="
);
}
#[tokio::test]
async fn a_placed_agent_gets_its_hive_a_password_and_the_archive_its_hash() {
let store = Memory::up();
let done = reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
assert_eq!(
done,
Done {
granted: 1,
revoked: 0,
failed: 0
}
);
let rest = store.value(ALPHA_ATLAS, REST_PASSWORD).expect("a password");
let restic = store.value(ALPHA_ATLAS, RESTIC_PASSWORD).expect("restic");
assert_eq!(
store.value(USER_ATLAS, HTPASSWD),
Some(htpasswd_line("atlas", &rest))
);
assert_eq!(store.value(RESTIC_ATLAS, PASSWORD), Some(restic));
assert_ne!(Some(rest), store.value(ALPHA_ATLAS, RESTIC_PASSWORD));
// The hash line is all the archive's user leaf holds.
assert!(!store.get(USER_ATLAS).unwrap().contains_key(REST_PASSWORD));
// Presence control for the absence checks below: only the placed hive.
assert!(store.get(BETA_ATLAS).is_none());
}
#[tokio::test]
async fn a_current_grant_is_not_rewritten() {
let store = Memory::up();
let one = placed(&[("alpha", "atlas")]);
reconcile(&store, &one, &none(), &hives()).await;
let before = store.get(ALPHA_ATLAS);
let done = reconcile(&store, &one, &none(), &hives()).await;
assert_eq!(done, Done::default());
assert_eq!(store.get(ALPHA_ATLAS), before);
}
#[tokio::test]
async fn a_move_replaces_the_password_and_takes_the_old_hives_copy() {
let store = Memory::up();
reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
let old_rest = store.value(ALPHA_ATLAS, REST_PASSWORD).unwrap();
let restic = store.value(RESTIC_ATLAS, PASSWORD).unwrap();
let done = reconcile(&store, &placed(&[("beta", "atlas")]), &none(), &hives()).await;
assert_eq!(done.granted, 1);
assert_eq!(done.revoked, 1);
assert!(store.get(ALPHA_ATLAS).is_none(), "the old hive loses read");
let new_rest = store.value(BETA_ATLAS, REST_PASSWORD).expect("new hive");
assert_ne!(new_rest, old_rest, "the old password stops working");
assert_eq!(
store.value(USER_ATLAS, HTPASSWD),
Some(htpasswd_line("atlas", &new_rest))
);
assert_ne!(
store.value(USER_ATLAS, HTPASSWD),
Some(htpasswd_line("atlas", &old_rest))
);
assert_eq!(
store.value(BETA_ATLAS, RESTIC_PASSWORD).as_deref(),
Some(restic.as_str()),
"the repository stays readable on the new hive"
);
}
#[tokio::test]
async fn a_destroyed_agent_loses_every_credential_but_the_restic_password() {
let store = Memory::up();
reconcile(
&store,
&placed(&[("alpha", "atlas"), ("beta", "argus")]),
&none(),
&hives(),
)
.await;
let done = reconcile(&store, &placed(&[("beta", "argus")]), &none(), &hives()).await;
assert_eq!(done.revoked, 2);
assert!(store.get(ALPHA_ATLAS).is_none());
assert!(store.get(USER_ATLAS).is_none());
assert!(
store.get(RESTIC_ATLAS).is_some(),
"the backups stay readable"
);
// Control: the agent still placed keeps all of it.
assert!(store.get("swarm/hives/beta/archive/argus").is_some());
assert!(store.get("swarm/services/archive/users/argus").is_some());
}
#[tokio::test]
async fn an_agent_two_hives_place_is_left_as_it_is() {
let store = Memory::up();
reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
let before = (store.get(ALPHA_ATLAS), store.get(USER_ATLAS));
let both = placed(&[("alpha", "atlas"), ("beta", "atlas")]);
let done = reconcile(&store, &both, &none(), &hives()).await;
assert_eq!(done, Done::default());
assert_eq!((store.get(ALPHA_ATLAS), store.get(USER_ATLAS)), before);
assert!(store.get(BETA_ATLAS).is_none());
// Neither hive is granted when the agent is new and both place it.
let fresh = Memory::up();
let done = reconcile(&fresh, &both, &none(), &hives()).await;
assert_eq!(done, Done::default());
assert_eq!(
fresh.kv.lock().unwrap().keys().collect::<Vec<_>>(),
[&archive::ready_path().unwrap()],
"only the archive's own marker is stored"
);
}
#[tokio::test]
async fn a_lost_user_line_is_rewritten_from_the_hives_password() {
let store = Memory::up();
let one = placed(&[("alpha", "atlas")]);
reconcile(&store, &one, &none(), &hives()).await;
let line = store.value(USER_ATLAS, HTPASSWD);
store.kv.lock().unwrap().remove(USER_ATLAS);
let done = reconcile(&store, &one, &none(), &hives()).await;
assert_eq!(done.granted, 1);
assert_eq!(store.value(USER_ATLAS, HTPASSWD), line);
}
#[tokio::test]
async fn a_refused_write_counts_as_failed_and_the_other_agents_still_get_theirs() {
let store = Memory {
refuse: Some("swarm/hives/alpha/"),
..Memory::up()
};
let done = reconcile(
&store,
&placed(&[("alpha", "atlas"), ("beta", "argus")]),
&none(),
&hives(),
)
.await;
assert_eq!(done.failed, 1);
assert_eq!(done.granted, 1);
assert!(store.get("swarm/hives/beta/archive/argus").is_some());
assert!(
store.get(USER_ATLAS).is_none(),
"no hash without the hive's copy"
);
}
#[tokio::test]
async fn the_restic_password_survives_a_revoke_and_a_new_grant_and_a_move() {
let store = Memory::up();
let atlas = |hive: &str| placed(&[(hive, "atlas")]);
reconcile(&store, &atlas("alpha"), &none(), &hives()).await;
let restic = store.value(RESTIC_ATLAS, PASSWORD).expect("minted once");
let rest = store.value(ALPHA_ATLAS, REST_PASSWORD).expect("granted");
// Nothing places the agent: every credential goes, the restic one stays.
let nowhere = placed(&[]);
reconcile(&store, &nowhere, &none(), &hives()).await;
assert!(store.get(ALPHA_ATLAS).is_none() && store.get(USER_ATLAS).is_none());
assert_eq!(store.value(RESTIC_ATLAS, PASSWORD).as_ref(), Some(&restic));
// Placed again: the login is new, the repository's password is the one it was created with.
reconcile(&store, &atlas("alpha"), &none(), &hives()).await;
assert_ne!(store.value(ALPHA_ATLAS, REST_PASSWORD), Some(rest));
assert_eq!(
store.value(ALPHA_ATLAS, RESTIC_PASSWORD).as_ref(),
Some(&restic)
);
assert_eq!(store.value(RESTIC_ATLAS, PASSWORD).as_ref(), Some(&restic));
// And across a move.
reconcile(&store, &atlas("beta"), &none(), &hives()).await;
assert_eq!(
store.value(BETA_ATLAS, RESTIC_PASSWORD).as_ref(),
Some(&restic)
);
assert_eq!(store.value(RESTIC_ATLAS, PASSWORD).as_ref(), Some(&restic));
}
#[tokio::test]
async fn a_hive_with_no_declaration_keeps_its_copies_and_its_agents_user_lines() {
let store = Memory::up();
reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
let (copy, line) = (store.get(ALPHA_ATLAS), store.get(USER_ATLAS));
// alpha's declaration is not there: it does not place atlas, and that proves nothing.
let gap: BTreeSet<String> = ["alpha".to_owned()].into();
let done = reconcile(&store, &placed(&[]), &gap, &hives()).await;
assert_eq!(done, Done::default());
assert_eq!(
(store.get(ALPHA_ATLAS), store.get(USER_ATLAS)),
(copy, line)
);
// Control: with the declaration read, the same placements revoke.
let done = reconcile(&store, &placed(&[]), &none(), &hives()).await;
assert_eq!(done.revoked, 2);
assert!(store.get(ALPHA_ATLAS).is_none() && store.get(USER_ATLAS).is_none());
}
#[tokio::test]
async fn a_swarm_whose_archive_has_not_announced_itself_gets_nothing() {
let store = Memory::default();
let done = reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
assert_eq!(done, Done::default());
assert!(store.kv.lock().unwrap().is_empty());
// Control: the same placements, once the archive is up.
store.kv.lock().unwrap().insert(
archive::ready_path().unwrap(),
fields(&[(archive::READY, "1")]),
);
let done = reconcile(&store, &placed(&[("alpha", "atlas")]), &none(), &hives()).await;
assert_eq!(done.granted, 1);
}
}