feat(#962): topology-driven child bind mounts + can_manage_top_level_agents role
Removes the MANAGER_NAME special-case from set_nspawn_flags in favour of two general mechanisms: 1. Topology-driven child mounts: every agent now gets its direct children's state, harness, and config dirs bind-mounted (RW). Root's children are the top-level agents, so root gets the same access it did before via the old /agents blob bind — but derived from topology, not a hardcoded name check. 2. can_manage_top_level_agents role: agents holding this role additionally get every top-level agent treated as a virtual child (same RW mounts) plus /applied and /meta as RO. Designed for recovery: a role holder can update a top-level agent's config even when that agent is down. Root receives this role by default on first reconcile_roles call. Operator can revoke it with set_role. Every agent (including root) now gets its own state/harness/config dirs via the standard path. Roles are stored in meta/roles.json (same dir as topology.json); reconcile_roles is called from reconcile so both files stay in sync.
This commit is contained in:
parent
ba43b5f869
commit
6d2d0ed847
2 changed files with 164 additions and 50 deletions
|
|
@ -1064,6 +1064,23 @@ const HOST_META_ROOT: &str = "/var/lib/hyperhive/meta";
|
||||||
/// Shared directory accessible to all agents. All agents bind-mount this RW.
|
/// Shared directory accessible to all agents. All agents bind-mount this RW.
|
||||||
const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared";
|
const HOST_SHARED_ROOT: &str = "/var/lib/hyperhive/shared";
|
||||||
|
|
||||||
|
/// Append bind flags for `child`'s state, harness, and config dirs into
|
||||||
|
/// `binds`. All three are RW so the parent can read/write state and
|
||||||
|
/// submit config-change requests. Creates missing host-side directories
|
||||||
|
/// so nspawn doesn't refuse to start; missing dirs are non-fatal.
|
||||||
|
fn bind_child_agent_dirs(child: &str, binds: &mut String) {
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
let state_dir = format!("{HOST_AGENTS_ROOT}/{child}/state");
|
||||||
|
let harness_dir = format!("{HOST_AGENTS_ROOT}/{child}/harness");
|
||||||
|
let config_dir = format!("{HOST_AGENTS_ROOT}/{child}/config");
|
||||||
|
for dir in [&state_dir, &harness_dir, &config_dir] {
|
||||||
|
let _ = std::fs::create_dir_all(dir);
|
||||||
|
}
|
||||||
|
let _ = write!(binds, " --bind={state_dir}:/agents/{child}/state");
|
||||||
|
let _ = write!(binds, " --bind={harness_dir}:/agents/{child}/harness");
|
||||||
|
let _ = write!(binds, " --bind={config_dir}:/agents/{child}/config");
|
||||||
|
}
|
||||||
|
|
||||||
fn set_nspawn_flags(
|
fn set_nspawn_flags(
|
||||||
container: &str,
|
container: &str,
|
||||||
runtime_dir: &Path,
|
runtime_dir: &Path,
|
||||||
|
|
@ -1097,21 +1114,17 @@ fn set_nspawn_flags(
|
||||||
shared = HOST_SHARED_ROOT,
|
shared = HOST_SHARED_ROOT,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Per-agent state + harness dirs. Skipped for the manager —
|
// Own state, harness, and config dirs — same for every agent including
|
||||||
// the `/agents` bind below already exposes both (along with
|
// root. Config is RO: an agent must not edit its own config; changes
|
||||||
// every sub-agent's). For regular agents the harness dir is
|
// only ever flow through the approval queue.
|
||||||
// the sibling of notes_dir (same parent, "harness" subdir).
|
{
|
||||||
if container != MANAGER_CONTAINER {
|
|
||||||
let _ = write!(
|
let _ = write!(
|
||||||
binds,
|
binds,
|
||||||
" --bind={notes}:/agents/{agent_name}/state",
|
" --bind={notes}:/agents/{agent_name}/state",
|
||||||
notes = notes_dir.display(),
|
notes = notes_dir.display(),
|
||||||
);
|
);
|
||||||
// Harness dir: sibling of notes_dir under the agent state root.
|
if let Some(state_parent) = notes_dir.parent() {
|
||||||
// systemd-nspawn refuses to start when the bind source is missing;
|
let harness_dir = state_parent.join("harness");
|
||||||
// ensure_state_dir already creates it, but be defensive here.
|
|
||||||
if let Some(parent) = notes_dir.parent() {
|
|
||||||
let harness_dir = parent.join("harness");
|
|
||||||
if !harness_dir.exists() {
|
if !harness_dir.exists() {
|
||||||
let _ = std::fs::create_dir_all(&harness_dir);
|
let _ = std::fs::create_dir_all(&harness_dir);
|
||||||
}
|
}
|
||||||
|
|
@ -1121,34 +1134,41 @@ fn set_nspawn_flags(
|
||||||
harness = harness_dir.display(),
|
harness = harness_dir.display(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
let own_config = format!("{HOST_AGENTS_ROOT}/{agent_name}/config");
|
||||||
|
std::fs::create_dir_all(&own_config)
|
||||||
|
.with_context(|| format!("create {own_config}"))?;
|
||||||
|
let _ = write!(binds, " --bind-ro={own_config}:/agents/{agent_name}/config");
|
||||||
}
|
}
|
||||||
if container == MANAGER_CONTAINER {
|
|
||||||
|
// Topology-driven child mounts: every direct child of this agent gets
|
||||||
|
// its state, harness, and config dirs bind-mounted RW so the parent
|
||||||
|
// can read state and manage config.
|
||||||
|
let direct_children = crate::topology::children_of(agent_name);
|
||||||
|
for child in &direct_children {
|
||||||
|
bind_child_agent_dirs(child, &mut binds);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `can_manage_top_level_agents` role: additionally mount every
|
||||||
|
// top-level agent (direct child of root) as a virtual child. Enables
|
||||||
|
// recovery — a role holder can update a top-level agent's config even
|
||||||
|
// when that agent is down. Also grants RO access to /applied and /meta.
|
||||||
|
if crate::topology::has_role(
|
||||||
|
agent_name,
|
||||||
|
crate::topology::ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS,
|
||||||
|
) {
|
||||||
|
let top_level = crate::topology::children_of(MANAGER_NAME);
|
||||||
|
for tl in &top_level {
|
||||||
|
if !direct_children.contains(tl) {
|
||||||
|
bind_child_agent_dirs(tl, &mut binds);
|
||||||
|
}
|
||||||
|
}
|
||||||
// systemd-nspawn refuses to start a container whose bind
|
// systemd-nspawn refuses to start a container whose bind
|
||||||
// source doesn't exist. The meta repo is created by the
|
// source doesn't exist. The meta repo is created by the
|
||||||
// startup migration, but make sure the directory is there
|
// startup migration, but make sure the directory is there
|
||||||
// before the manager comes up in case set_nspawn_flags fires
|
// before the role holder comes up in case set_nspawn_flags
|
||||||
// first (e.g. cold start with no agents).
|
// fires first (e.g. cold start with no agents).
|
||||||
std::fs::create_dir_all(HOST_META_ROOT)
|
std::fs::create_dir_all(HOST_META_ROOT)
|
||||||
.with_context(|| format!("create {HOST_META_ROOT}"))?;
|
.with_context(|| format!("create {HOST_META_ROOT}"))?;
|
||||||
// Manager edits sub-agent proposed/ repos and its own. RW so it can
|
|
||||||
// git-commit. Sub-agents see only their own /run/hive socket and
|
|
||||||
// /root/.claude (no /agents or /applied).
|
|
||||||
//
|
|
||||||
// /applied is a separate RO mount of the hive-c0re-only applied
|
|
||||||
// repos so the manager can `git fetch /applied/<n>/.git
|
|
||||||
// refs/tags/*:refs/tags/applied/*` to mirror deployed/failed/
|
|
||||||
// denied tags into its proposed clones and diff against
|
|
||||||
// what's actually deployed. RO bind makes destructive git
|
|
||||||
// plumbing inside the container unable to corrupt applied.
|
|
||||||
//
|
|
||||||
// /meta is a third RO mount exposing the system-wide deploy
|
|
||||||
// flake (`git log /meta --oneline` shows every deploy across
|
|
||||||
// every agent; `cat /meta/flake.lock` resolves which sha each
|
|
||||||
// agent is pinned at right now).
|
|
||||||
let _ = write!(
|
|
||||||
binds,
|
|
||||||
" --bind={HOST_AGENTS_ROOT}:{CONTAINER_MANAGER_AGENTS_MOUNT}",
|
|
||||||
);
|
|
||||||
let _ = write!(
|
let _ = write!(
|
||||||
binds,
|
binds,
|
||||||
" --bind-ro={HOST_APPLIED_ROOT}:{CONTAINER_MANAGER_APPLIED_MOUNT}",
|
" --bind-ro={HOST_APPLIED_ROOT}:{CONTAINER_MANAGER_APPLIED_MOUNT}",
|
||||||
|
|
@ -1158,24 +1178,6 @@ fn set_nspawn_flags(
|
||||||
" --bind-ro={HOST_META_ROOT}:{mount}",
|
" --bind-ro={HOST_META_ROOT}:{mount}",
|
||||||
mount = crate::meta::CONTAINER_MANAGER_META_MOUNT,
|
mount = crate::meta::CONTAINER_MANAGER_META_MOUNT,
|
||||||
);
|
);
|
||||||
} else {
|
|
||||||
// Sub-agents get a READ-ONLY view of their own proposed
|
|
||||||
// config repo at /agents/<name>/config — agent.nix plus
|
|
||||||
// whatever extra files the manager split the config into.
|
|
||||||
// Lets an agent inspect exactly what defines it and request
|
|
||||||
// precise changes from the manager. RO is load-bearing: the
|
|
||||||
// agent must NOT edit its own config — changes only ever flow
|
|
||||||
// through the manager's RW proposed repo + the approval
|
|
||||||
// queue. The manager already has this dir RW via the /agents
|
|
||||||
// tree bind above.
|
|
||||||
let config_dir = format!("{HOST_AGENTS_ROOT}/{agent_name}/config");
|
|
||||||
// nspawn refuses to start when a bind source is missing.
|
|
||||||
// `setup_proposed` seeds this dir before spawn reaches here,
|
|
||||||
// but create defensively so a missing repo degrades to an
|
|
||||||
// empty RO dir instead of a container that won't boot.
|
|
||||||
std::fs::create_dir_all(&config_dir).with_context(|| format!("create {config_dir}"))?;
|
|
||||||
let _ = write!(binds, " --bind-ro={config_dir}:/agents/{agent_name}/config");
|
|
||||||
|
|
||||||
}
|
}
|
||||||
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
// Web-socket subdir: bind-mount `/run/hive-agent/<name>/` into the
|
||||||
// container so the harness can bind `web.sock` there and the host-side
|
// container so the harness can bind `web.sock` there and the host-side
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,10 @@
|
||||||
//! alongside the meta `flake.nix`, so topology changes thread through
|
//! alongside the meta `flake.nix`, so topology changes thread through
|
||||||
//! the same git commit log as deploys.
|
//! the same git commit log as deploys.
|
||||||
//!
|
//!
|
||||||
|
//! Agent roles are stored alongside in `roles.json` as a flat map of
|
||||||
|
//! `name → [role, ...]`. Roles gate additional bind-mount grants; see
|
||||||
|
//! `lifecycle::set_nspawn_flags` for the consumer.
|
||||||
|
//!
|
||||||
//! Format, rationale, read/reconcile/inject/surface flow, and target
|
//! Format, rationale, read/reconcile/inject/surface flow, and target
|
||||||
//! enforcement semantics: `docs/agent-hierarchy.md::Current state`.
|
//! enforcement semantics: `docs/agent-hierarchy.md::Current state`.
|
||||||
//! `<parent>` sentinel resolution (delivered by [`resolve_recipient`]):
|
//! `<parent>` sentinel resolution (delivered by [`resolve_recipient`]):
|
||||||
|
|
@ -278,6 +282,114 @@ pub fn reconcile(agent_names: &[String]) -> std::io::Result<bool> {
|
||||||
if changed {
|
if changed {
|
||||||
write(¤t)?;
|
write(¤t)?;
|
||||||
}
|
}
|
||||||
|
// Keep roles in sync with the agent set.
|
||||||
|
reconcile_roles(agent_names)?;
|
||||||
|
Ok(changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Roles
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Agents with this role have the top-level agents (direct children of the
|
||||||
|
/// root/manager agent) added as virtual children for bind-mount and
|
||||||
|
/// config-change purposes. Enables recovery: if a top-level agent is down,
|
||||||
|
/// a role holder can still read its state and update its config.
|
||||||
|
///
|
||||||
|
/// The root agent receives this role by default on first `reconcile_roles`
|
||||||
|
/// call; operators can revoke it with `set_role`.
|
||||||
|
pub const ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS: &str = "can_manage_top_level_agents";
|
||||||
|
|
||||||
|
const ROLES_FILE: &str = "roles.json";
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub fn roles_path() -> std::path::PathBuf {
|
||||||
|
crate::meta::meta_dir().join(ROLES_FILE)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the roles map from disk. Returns an empty map when absent or
|
||||||
|
/// unparsable — same safe-degradation pattern as `topology::read`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn read_roles() -> BTreeMap<String, Vec<String>> {
|
||||||
|
let Ok(raw) = std::fs::read_to_string(roles_path()) else {
|
||||||
|
return BTreeMap::new();
|
||||||
|
};
|
||||||
|
serde_json::from_str(&raw).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the roles map. Sorted output keeps git diffs minimal.
|
||||||
|
pub fn write_roles(roles: &BTreeMap<String, Vec<String>>) -> std::io::Result<()> {
|
||||||
|
let path = roles_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
let text = serde_json::to_string_pretty(roles)
|
||||||
|
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||||
|
std::fs::write(&path, format!("{text}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return true when `name` holds `role`.
|
||||||
|
#[must_use]
|
||||||
|
pub fn has_role(name: &str, role: &str) -> bool {
|
||||||
|
has_role_in(&read_roles(), name, role)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure form of [`has_role`] for unit tests.
|
||||||
|
#[must_use]
|
||||||
|
pub fn has_role_in(roles: &BTreeMap<String, Vec<String>>, name: &str, role: &str) -> bool {
|
||||||
|
roles
|
||||||
|
.get(name)
|
||||||
|
.is_some_and(|rs| rs.iter().any(|r| r == role))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Grant or revoke a role for `name`. Idempotent — no disk write when the
|
||||||
|
/// state is already correct.
|
||||||
|
pub fn set_role(name: &str, role: &str, enabled: bool) -> Result<(), String> {
|
||||||
|
let mut roles = read_roles();
|
||||||
|
let list = roles.entry(name.to_owned()).or_default();
|
||||||
|
let held = list.iter().any(|r| r == role);
|
||||||
|
match (enabled, held) {
|
||||||
|
(true, false) => list.push(role.to_owned()),
|
||||||
|
(false, true) => list.retain(|r| r != role),
|
||||||
|
_ => return Ok(()),
|
||||||
|
}
|
||||||
|
if roles.get(name).is_some_and(|l| l.is_empty()) {
|
||||||
|
roles.remove(name);
|
||||||
|
}
|
||||||
|
write_roles(&roles).map_err(|e| format!("write roles.json: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reconcile `roles.json` against the current agent set:
|
||||||
|
/// - Seeds root's default `can_manage_top_level_agents` role on first
|
||||||
|
/// appearance (operator can revoke with `set_role`).
|
||||||
|
/// - Drops entries for agents that no longer exist.
|
||||||
|
///
|
||||||
|
/// Returns true when the file changed.
|
||||||
|
pub fn reconcile_roles(agent_names: &[String]) -> std::io::Result<bool> {
|
||||||
|
let mut roles = read_roles();
|
||||||
|
let mut changed = false;
|
||||||
|
|
||||||
|
let root = crate::lifecycle::MANAGER_NAME;
|
||||||
|
if agent_names.iter().any(|n| n == root) && !roles.contains_key(root) {
|
||||||
|
roles.insert(
|
||||||
|
root.to_owned(),
|
||||||
|
vec![ROLE_CAN_MANAGE_TOP_LEVEL_AGENTS.to_owned()],
|
||||||
|
);
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let known: std::collections::HashSet<_> = agent_names.iter().collect();
|
||||||
|
roles.retain(|name, _| {
|
||||||
|
let keep = known.contains(name);
|
||||||
|
if !keep {
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
|
||||||
|
if changed {
|
||||||
|
write_roles(&roles)?;
|
||||||
|
}
|
||||||
Ok(changed)
|
Ok(changed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue