feat(#1184): make agent CPU quota and memory limit configurable
This commit is contained in:
parent
4f80253101
commit
d1fbb4aef8
7 changed files with 80 additions and 12 deletions
|
|
@ -237,6 +237,8 @@ pub async fn run_approval_spawn(
|
||||||
coord.dashboard_port,
|
coord.dashboard_port,
|
||||||
&coord.operator_pronouns,
|
&coord.operator_pronouns,
|
||||||
&coord.context_window_tokens,
|
&coord.context_window_tokens,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
if result.is_ok() {
|
if result.is_ok() {
|
||||||
|
|
@ -589,6 +591,8 @@ async fn run_apply_commit(
|
||||||
applied_dir,
|
applied_dir,
|
||||||
claude_dir,
|
claude_dir,
|
||||||
notes_dir,
|
notes_dir,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
&|step| coord.set_queue_step(queue_entry_id, step),
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,8 @@ pub async fn rebuild_agent(
|
||||||
coord.dashboard_port,
|
coord.dashboard_port,
|
||||||
&coord.operator_pronouns,
|
&coord.operator_pronouns,
|
||||||
&coord.context_window_tokens,
|
&coord.context_window_tokens,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
&|step| coord.set_queue_step(queue_entry_id, step),
|
&|step| coord.set_queue_step(queue_entry_id, step),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
@ -205,6 +207,8 @@ pub async fn ensure_root_agent(coord: &Arc<Coordinator>) -> Result<()> {
|
||||||
coord.dashboard_port,
|
coord.dashboard_port,
|
||||||
&coord.operator_pronouns,
|
&coord.operator_pronouns,
|
||||||
&coord.context_window_tokens,
|
&coord.context_window_tokens,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
if let Some(rev) = current_rev {
|
if let Some(rev) = current_rev {
|
||||||
|
|
|
||||||
|
|
@ -81,6 +81,11 @@ pub struct Coordinator {
|
||||||
/// compaction / auto-reset watermarks and exposes the active value
|
/// compaction / auto-reset watermarks and exposes the active value
|
||||||
/// on `/api/state` as `context_window_tokens`.
|
/// on `/api/state` as `context_window_tokens`.
|
||||||
pub context_window_tokens: std::collections::HashMap<String, u64>,
|
pub context_window_tokens: std::collections::HashMap<String, u64>,
|
||||||
|
/// Per-agent systemd `CPUQuota=` value (e.g. `"200%"`). Written into
|
||||||
|
/// the `container@h-<name>.service.d/` drop-in on every spawn/rebuild.
|
||||||
|
pub agent_cpu_quota: String,
|
||||||
|
/// Per-agent systemd `MemoryMax=` value (e.g. `"4G"`). Same drop-in.
|
||||||
|
pub agent_memory_max: String,
|
||||||
agents: Mutex<HashMap<String, AgentSocket>>,
|
agents: Mutex<HashMap<String, AgentSocket>>,
|
||||||
/// Agents whose lifecycle action (currently just spawn) is in flight.
|
/// Agents whose lifecycle action (currently just spawn) is in flight.
|
||||||
/// Read by the dashboard to render a spinner; cleared when the action
|
/// Read by the dashboard to render a spinner; cleared when the action
|
||||||
|
|
@ -228,6 +233,8 @@ impl Coordinator {
|
||||||
dashboard_port: u16,
|
dashboard_port: u16,
|
||||||
operator_pronouns: String,
|
operator_pronouns: String,
|
||||||
context_window_tokens: std::collections::HashMap<String, u64>,
|
context_window_tokens: std::collections::HashMap<String, u64>,
|
||||||
|
agent_cpu_quota: String,
|
||||||
|
agent_memory_max: String,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let broker = Broker::open(db_path).context("open broker")?;
|
let broker = Broker::open(db_path).context("open broker")?;
|
||||||
let approvals = Approvals::open(db_path).context("open approvals")?;
|
let approvals = Approvals::open(db_path).context("open approvals")?;
|
||||||
|
|
@ -261,6 +268,8 @@ impl Coordinator {
|
||||||
dashboard_port,
|
dashboard_port,
|
||||||
operator_pronouns,
|
operator_pronouns,
|
||||||
context_window_tokens,
|
context_window_tokens,
|
||||||
|
agent_cpu_quota,
|
||||||
|
agent_memory_max,
|
||||||
agents: Mutex::new(HashMap::new()),
|
agents: Mutex::new(HashMap::new()),
|
||||||
transient: Mutex::new(HashMap::new()),
|
transient: Mutex::new(HashMap::new()),
|
||||||
recent_transient: Mutex::new(HashMap::new()),
|
recent_transient: Mutex::new(HashMap::new()),
|
||||||
|
|
|
||||||
|
|
@ -50,10 +50,6 @@ const GIT_EMAIL: &str = "c0re@hyperhive.local";
|
||||||
const WEB_PORT_BASE: u16 = 8100;
|
const WEB_PORT_BASE: u16 = 8100;
|
||||||
const WEB_PORT_RANGE: u16 = 900;
|
const WEB_PORT_RANGE: u16 = 900;
|
||||||
|
|
||||||
/// Default resource caps applied to every managed container via a systemd
|
|
||||||
/// drop-in under `/run/systemd/system/container@<NAME>.service.d/`.
|
|
||||||
const DEFAULT_MEMORY_MAX: &str = "2G";
|
|
||||||
const DEFAULT_CPU_QUOTA: &str = "50%";
|
|
||||||
|
|
||||||
/// FNV-1a hash of a string — shared by `agent_web_port` and
|
/// FNV-1a hash of a string — shared by `agent_web_port` and
|
||||||
/// `agent_network_ip` so the derivation rule is identical.
|
/// `agent_network_ip` so the derivation rule is identical.
|
||||||
|
|
@ -240,6 +236,8 @@ pub async fn spawn(
|
||||||
dashboard_port: u16,
|
dashboard_port: u16,
|
||||||
operator_pronouns: &str,
|
operator_pronouns: &str,
|
||||||
context_window_tokens: &std::collections::HashMap<String, u64>,
|
context_window_tokens: &std::collections::HashMap<String, u64>,
|
||||||
|
cpu_quota: &str,
|
||||||
|
memory_max: &str,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
validate(name)?;
|
validate(name)?;
|
||||||
if let Some(other) = port_collision(name).await {
|
if let Some(other) = port_collision(name).await {
|
||||||
|
|
@ -269,7 +267,7 @@ pub async fn spawn(
|
||||||
let container = container_name(name);
|
let container = container_name(name);
|
||||||
priv_run("create", name).await?;
|
priv_run("create", name).await?;
|
||||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
||||||
set_resource_limits(&container).await?;
|
set_resource_limits(&container, memory_max, cpu_quota).await?;
|
||||||
systemd_daemon_reload().await?;
|
systemd_daemon_reload().await?;
|
||||||
priv_run("start", name).await
|
priv_run("start", name).await
|
||||||
}
|
}
|
||||||
|
|
@ -398,6 +396,8 @@ pub async fn rebuild(
|
||||||
dashboard_port: u16,
|
dashboard_port: u16,
|
||||||
operator_pronouns: &str,
|
operator_pronouns: &str,
|
||||||
context_window_tokens: &std::collections::HashMap<String, u64>,
|
context_window_tokens: &std::collections::HashMap<String, u64>,
|
||||||
|
cpu_quota: &str,
|
||||||
|
memory_max: &str,
|
||||||
on_step: &(dyn Fn(&str) + Send + Sync),
|
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Sync the meta flake (idempotent — no-op when the rendered
|
// Sync the meta flake (idempotent — no-op when the rendered
|
||||||
|
|
@ -420,7 +420,7 @@ pub async fn rebuild(
|
||||||
// `applied/<n>/main` currently points at (deployed/<latest>).
|
// `applied/<n>/main` currently points at (deployed/<latest>).
|
||||||
// Commits the lock if it changed.
|
// Commits the lock if it changed.
|
||||||
crate::meta::lock_update_for_rebuild(name).await?;
|
crate::meta::lock_update_for_rebuild(name).await?;
|
||||||
rebuild_no_meta(name, agent_dir, applied_dir, claude_dir, notes_dir, on_step).await
|
rebuild_no_meta(name, agent_dir, applied_dir, claude_dir, notes_dir, cpu_quota, memory_max, on_step).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Container-level rebuild without touching the meta repo. Callers
|
/// Container-level rebuild without touching the meta repo. Callers
|
||||||
|
|
@ -439,6 +439,8 @@ pub async fn rebuild_no_meta(
|
||||||
applied_dir: &Path,
|
applied_dir: &Path,
|
||||||
claude_dir: &Path,
|
claude_dir: &Path,
|
||||||
notes_dir: &Path,
|
notes_dir: &Path,
|
||||||
|
cpu_quota: &str,
|
||||||
|
memory_max: &str,
|
||||||
on_step: &(dyn Fn(&str) + Send + Sync),
|
on_step: &(dyn Fn(&str) + Send + Sync),
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
validate(name)?;
|
validate(name)?;
|
||||||
|
|
@ -458,7 +460,7 @@ pub async fn rebuild_no_meta(
|
||||||
// See `docs/coordinator.md::Container lifecycle`.
|
// See `docs/coordinator.md::Container lifecycle`.
|
||||||
let was_running = is_running(name).await;
|
let was_running = is_running(name).await;
|
||||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
||||||
set_resource_limits(&container).await?;
|
set_resource_limits(&container, memory_max, cpu_quota).await?;
|
||||||
systemd_daemon_reload().await?;
|
systemd_daemon_reload().await?;
|
||||||
if was_running {
|
if was_running {
|
||||||
on_step("nix build");
|
on_step("nix build");
|
||||||
|
|
@ -530,7 +532,7 @@ pub async fn rebuild_no_meta(
|
||||||
on_step("nixos-container create");
|
on_step("nixos-container create");
|
||||||
priv_run("create", name).await?;
|
priv_run("create", name).await?;
|
||||||
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
set_nspawn_flags(&container, agent_dir, claude_dir, notes_dir).await?;
|
||||||
set_resource_limits(&container).await?;
|
set_resource_limits(&container, memory_max, cpu_quota).await?;
|
||||||
systemd_daemon_reload().await?;
|
systemd_daemon_reload().await?;
|
||||||
on_step("nixos-container start");
|
on_step("nixos-container start");
|
||||||
priv_run("start", name).await
|
priv_run("start", name).await
|
||||||
|
|
@ -1049,9 +1051,8 @@ pub async fn git_update_ref(dir: &Path, refname: &str, target: &str) -> Result<(
|
||||||
/// Write a systemd drop-in for `container@<container>.service` that applies
|
/// Write a systemd drop-in for `container@<container>.service` that applies
|
||||||
/// our default resource caps. Goes under `/run/systemd/system/...` so it's
|
/// our default resource caps. Goes under `/run/systemd/system/...` so it's
|
||||||
/// ephemeral (regenerated on every spawn / rebuild).
|
/// ephemeral (regenerated on every spawn / rebuild).
|
||||||
async fn set_resource_limits(container: &str) -> Result<()> {
|
async fn set_resource_limits(container: &str, memory_max: &str, cpu_quota: &str) -> Result<()> {
|
||||||
crate::priv_client::write_resource_limits(container, DEFAULT_MEMORY_MAX, DEFAULT_CPU_QUOTA)
|
crate::priv_client::write_resource_limits(container, memory_max, cpu_quota).await
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn systemd_daemon_reload() -> Result<()> {
|
async fn systemd_daemon_reload() -> Result<()> {
|
||||||
|
|
|
||||||
|
|
@ -73,6 +73,15 @@ enum Cmd {
|
||||||
default_value = r#"{"haiku":200000,"sonnet":1000000,"opus":1000000}"#
|
default_value = r#"{"haiku":200000,"sonnet":1000000,"opus":1000000}"#
|
||||||
)]
|
)]
|
||||||
context_window_tokens: String,
|
context_window_tokens: String,
|
||||||
|
/// systemd `CPUQuota=` applied to every agent container via a drop-in.
|
||||||
|
/// Expressed as a percentage of one CPU core (e.g. `"200%"` = 2 cores).
|
||||||
|
/// Set via `services.hyperhive.agentCpuQuota`.
|
||||||
|
#[arg(long, default_value = "200%")]
|
||||||
|
agent_cpu_quota: String,
|
||||||
|
/// systemd `MemoryMax=` applied to every agent container.
|
||||||
|
/// Set via `services.hyperhive.agentMemoryMax`.
|
||||||
|
#[arg(long, default_value = "4G")]
|
||||||
|
agent_memory_max: String,
|
||||||
},
|
},
|
||||||
/// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
|
/// Spawn a new agent container directly (`hive-agent-<name>`). Bypasses
|
||||||
/// the approval queue — use only as an operator on the host. For
|
/// the approval queue — use only as an operator on the host. For
|
||||||
|
|
@ -141,6 +150,8 @@ async fn main() -> Result<()> {
|
||||||
dashboard_port,
|
dashboard_port,
|
||||||
operator_pronouns,
|
operator_pronouns,
|
||||||
context_window_tokens,
|
context_window_tokens,
|
||||||
|
agent_cpu_quota,
|
||||||
|
agent_memory_max,
|
||||||
} => {
|
} => {
|
||||||
cmd_serve(
|
cmd_serve(
|
||||||
hyperhive_flake,
|
hyperhive_flake,
|
||||||
|
|
@ -150,6 +161,8 @@ async fn main() -> Result<()> {
|
||||||
dashboard_port,
|
dashboard_port,
|
||||||
operator_pronouns,
|
operator_pronouns,
|
||||||
context_window_tokens,
|
context_window_tokens,
|
||||||
|
agent_cpu_quota,
|
||||||
|
agent_memory_max,
|
||||||
&cli.socket,
|
&cli.socket,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|
@ -199,6 +212,8 @@ async fn cmd_serve(
|
||||||
dashboard_port: u16,
|
dashboard_port: u16,
|
||||||
operator_pronouns: String,
|
operator_pronouns: String,
|
||||||
context_window_tokens: String,
|
context_window_tokens: String,
|
||||||
|
agent_cpu_quota: String,
|
||||||
|
agent_memory_max: String,
|
||||||
socket: &std::path::Path,
|
socket: &std::path::Path,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let cwt: std::collections::HashMap<String, u64> = serde_json::from_str(&context_window_tokens)
|
let cwt: std::collections::HashMap<String, u64> = serde_json::from_str(&context_window_tokens)
|
||||||
|
|
@ -211,6 +226,8 @@ async fn cmd_serve(
|
||||||
dashboard_port,
|
dashboard_port,
|
||||||
operator_pronouns,
|
operator_pronouns,
|
||||||
cwt,
|
cwt,
|
||||||
|
agent_cpu_quota,
|
||||||
|
agent_memory_max,
|
||||||
)?);
|
)?);
|
||||||
manager_server::start(coord.clone())?;
|
manager_server::start(coord.clone())?;
|
||||||
// Idempotent pre-flight: rewrite pre-meta-layout applied
|
// Idempotent pre-flight: rewrite pre-meta-layout applied
|
||||||
|
|
|
||||||
|
|
@ -97,6 +97,8 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
||||||
coord.dashboard_port,
|
coord.dashboard_port,
|
||||||
&coord.operator_pronouns,
|
&coord.operator_pronouns,
|
||||||
&coord.context_window_tokens,
|
&coord.context_window_tokens,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
|
@ -191,6 +193,8 @@ async fn dispatch(req: &HostRequest, coord: Arc<Coordinator>) -> HostResponse {
|
||||||
coord.dashboard_port,
|
coord.dashboard_port,
|
||||||
&coord.operator_pronouns,
|
&coord.operator_pronouns,
|
||||||
&coord.context_window_tokens,
|
&coord.context_window_tokens,
|
||||||
|
&coord.agent_cpu_quota,
|
||||||
|
&coord.agent_memory_max,
|
||||||
&|_| (),
|
&|_| (),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
|
||||||
|
|
@ -414,6 +414,35 @@ in
|
||||||
on the next `↻ R3BU1LD` — no per-agent approval needed.
|
on the next `↻ R3BU1LD` — no per-agent approval needed.
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
agentCpuQuota = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "200%";
|
||||||
|
example = "400%";
|
||||||
|
description = ''
|
||||||
|
systemd `CPUQuota=` applied to every agent container via a
|
||||||
|
`container@h-<name>.service.d/` drop-in written on each
|
||||||
|
spawn/rebuild. Expressed as a percentage of one CPU core —
|
||||||
|
`"200%"` allows each agent to use up to 2 cores. The old
|
||||||
|
hard-coded value was `"50%"`; bump this if agents are hitting
|
||||||
|
CPU limits during builds or heavy tool use.
|
||||||
|
|
||||||
|
For a hive-wide cap across all containers, set
|
||||||
|
`systemd.slices.machine.serviceConfig.CPUQuota` in your NixOS
|
||||||
|
config (all nspawn containers live in `machine.slice`).
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
agentMemoryMax = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "4G";
|
||||||
|
example = "8G";
|
||||||
|
description = ''
|
||||||
|
systemd `MemoryMax=` applied to every agent container via the
|
||||||
|
same drop-in as `agentCpuQuota`. The old hard-coded value was
|
||||||
|
`"2G"`.
|
||||||
|
'';
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
config = lib.mkIf cfg.enable {
|
||||||
|
|
@ -616,7 +645,7 @@ in
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --hyperhive-flake ${cfg.hyperhiveFlake} --nixpkgs-flake ${cfg.nixpkgsFlake} --nixpkgs-unstable-flake ${cfg.nixpkgsUnstableFlake} --dashboard-port ${toString cfg.dashboardPort} --operator-pronouns ${lib.escapeShellArg cfg.operatorPronouns} --context-window-tokens ${lib.escapeShellArg (builtins.toJSON cfg.contextWindowTokens)}";
|
ExecStart = "${cfg.package}/bin/hive-c0re --socket /run/hyperhive/host.sock serve --hyperhive-flake ${cfg.hyperhiveFlake} --nixpkgs-flake ${cfg.nixpkgsFlake} --nixpkgs-unstable-flake ${cfg.nixpkgsUnstableFlake} --dashboard-port ${toString cfg.dashboardPort} --operator-pronouns ${lib.escapeShellArg cfg.operatorPronouns} --context-window-tokens ${lib.escapeShellArg (builtins.toJSON cfg.contextWindowTokens)} --agent-cpu-quota ${lib.escapeShellArg cfg.agentCpuQuota} --agent-memory-max ${lib.escapeShellArg cfg.agentMemoryMax}";
|
||||||
# Migrate hive-c0re's *own* state to the service user after an
|
# Migrate hive-c0re's *own* state to the service user after an
|
||||||
# upgrade from a root-run install (systemd's StateDirectory only
|
# upgrade from a root-run install (systemd's StateDirectory only
|
||||||
# chowns the top-level dir, not pre-existing files inside it). The
|
# chowns the top-level dir, not pre-existing files inside it). The
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue