budgeted agents throttle at 90% of the cap; alert on repeated oom kills

hive-c0re/src/lifecycle/host_config.rs

@@ -25,13 +25,14 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
     validate(name)?;
     let container = container_name(name);
     set_nspawn_flags(&container, hive, &paths.agent, &paths.claude, &paths.notes).await?;
-    if let Some((cpu_quota, memory_max)) = stage_limits(
+    let host = HostMemory::read();
+    if let Some((cpu_quota, memory_max, memory_high)) = stage_limits(
         name,
         &crate::resource_limits::swarm_limits_path(),
         &limits_dropin_path(&container),
         &paths.agent,
         hive,
-        HostMemory::read(),
+        host,
     )? {
         set_resource_limits(
             &container,
@@ -39,6 +40,7 @@ pub async fn write_dropins(name: &str, hive: &HiveEnv, paths: &AgentPaths) -> Re
             &memory_max,
             hive.agent_cpu_weight,
             hive.agent_io_weight,
+            memory_high,
         )
         .await?;
     }
@@ -94,6 +96,8 @@ impl HostMemory {
 /// systemd enforces on it, so this file is how it learns the value it sizes
 /// its heap and its subagents against. It is written on every call, from the
 /// `MemoryMax=` the agent will start under: the new one, or the kept drop-in's.
+///
+/// The limits to apply come with the `MemoryHigh=` byte count for their cap.
 fn stage_limits(
     agent_name: &str,
     swarm_path: &Path,
@@ -101,7 +105,7 @@ fn stage_limits(
     runtime_dir: &Path,
     hive: &HiveEnv,
     host: Option<HostMemory>,
-) -> Result<Option<(String, String)>> {
+) -> Result<Option<(String, String, Option<u64>)>> {
     let limits = limits_to_apply(agent_name, swarm_path, dropin_path, hive);
     let memory_max = match &limits {
         Some((_, memory_max)) => Some(memory_max.clone()),
@@ -109,7 +113,10 @@ fn stage_limits(
     };
     let cap = memory_max.and_then(|m| memory_cap_value(agent_name, &m, host));
     write_memory_cap(runtime_dir, cap.as_deref())?;
-    Ok(limits)
+    Ok(limits.map(|(cpu_quota, memory_max)| {
+        let high = memory_high(agent_name, &memory_max, host);
+        (cpu_quota, memory_max, high)
+    }))
 }
 
 /// The `MemoryMax=` the drop-in at `path` applies: its last assignment, as
@@ -169,6 +176,16 @@ fn memory_cap_value(
     )
 }
 
+/// The `MemoryHigh=` byte count for an agent capped at `memory_max`: 90% of the
+/// cap, rounded down, so the kernel slows the container and reclaims from it
+/// before `MemoryMax=` kills in it. `None` when the agent has no cap.
+fn memory_high(agent_name: &str, memory_max: &str, host: Option<HostMemory>) -> Option<u64> {
+    let cap = memory_cap_value(agent_name, memory_max, host)?
+        .parse::<u64>()
+        .ok()?;
+    u64::try_from(u128::from(cap) * 9 / 10).ok()
+}
+
 /// `percent` (the part before `%`) in hundredths of a percent, with systemd's
 /// `parse_permyriad` rules: digits, at most two decimals, at most 100%.
 fn permyriad(percent: &str) -> Option<u64> {
@@ -277,9 +294,15 @@ async fn set_resource_limits(
     memory_max: &str,
     cpu_weight: Option<u32>,
     io_weight: Option<u32>,
+    memory_high: Option<u64>,
 ) -> Result<()> {
     crate::priv_client::write_resource_limits(
-        container, memory_max, cpu_quota, cpu_weight, io_weight,
+        container,
+        memory_max,
+        cpu_quota,
+        cpu_weight,
+        io_weight,
+        memory_high,
     )
     .await
 }
@@ -577,7 +600,7 @@ mod tests {
     use super::{
         BindMount, HostMemory, MEMORY_CAP_FILE, base_binds, bind_child_agent_dirs,
         bind_own_agent_dirs, holds_manage_root_agent, limits_to_apply, memory_cap_value,
-        stage_limits,
+        memory_high, stage_limits,
     };
     use crate::coordinator::HiveEnv;
     use hive_types::NixBuilds;
@@ -893,7 +916,7 @@ mod tests {
         )
         .expect("stage limits");
         let copy = std::fs::read_to_string(runtime.join(MEMORY_CAP_FILE)).ok();
-        (applied.map(|(_, mem)| mem), copy)
+        (applied.map(|(_, mem, _)| mem), copy)
     }
 
     #[test]
@@ -969,6 +992,52 @@ mod tests {
         );
     }
 
+    #[test]
+    fn memory_high_is_ninety_percent_of_the_cap_rounded_down() {
+        for (memory_max, high) in [
+            ("4G", Some(3_865_470_566)),
+            ("1048576B", Some(943_718)),
+            ("50%", Some(7_372_803_686)),
+            // No cap, or a value systemd ignores: nothing to throttle below.
+            ("infinity", None),
+            ("0", None),
+            ("150%", None),
+            ("lots", None),
+        ] {
+            assert_eq!(
+                memory_high("sock", memory_max, Some(HOST)),
+                high,
+                "{memory_max}"
+            );
+        }
+        // A percentage needs the host's memory; a size does not.
+        assert_eq!(memory_high("sock", "50%", None), None);
+        assert_eq!(memory_high("sock", "8G", None), Some(7_730_941_132));
+    }
+
+    #[test]
+    fn staged_limits_carry_the_throttle_point_of_their_cap() {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let swarm = swarm_file(&dir);
+        let staged = |dir: &tempfile::TempDir| {
+            stage_limits(
+                "sock",
+                &swarm,
+                &dir.path().join("hyperhive-limits.conf"),
+                &dir.path().join("runtime"),
+                &HiveEnv::default(),
+                Some(HOST),
+            )
+            .expect("stage limits")
+            .map(|(_, mem, high)| (mem, high))
+        };
+        // The hive default, 4G.
+        assert_eq!(staged(&dir), Some(("4G".to_owned(), Some(3_865_470_566))));
+        // Presence control above; an uncapped agent has no throttle point.
+        std::fs::write(&swarm, r#"{"sock": {"memory_max": "infinity"}}"#).expect("seed swarm caps");
+        assert_eq!(staged(&dir), Some(("infinity".to_owned(), None)));
+    }
+
     #[test]
     fn the_host_memory_systemd_would_read_is_found() {
         let host = HostMemory::read().expect("sysconf");
@@ -1027,7 +1096,7 @@ mod tests {
         let runtime = dir.path().join("runtime");
         let applied = stage_limits("sock", &swarm, &dropin, &runtime, &HiveEnv::default(), None)
             .expect("stage limits");
-        assert_eq!(applied.map(|(_, mem)| mem), Some("50%".to_owned()));
+        assert_eq!(applied.map(|(_, mem, _)| mem), Some("50%".to_owned()));
         assert!(!runtime.join(MEMORY_CAP_FILE).exists());
     }
 }

hive-c0re/src/priv_client.rs

@@ -292,9 +292,15 @@ pub async fn write_resource_limits(
     cpu_quota: &str,
     cpu_weight: Option<u32>,
     io_weight: Option<u32>,
+    memory_high: Option<u64>,
 ) -> Result<()> {
     ok(call(&resource_limits_request(
-        container, memory_max, cpu_quota, cpu_weight, io_weight,
+        container,
+        memory_max,
+        cpu_quota,
+        cpu_weight,
+        io_weight,
+        memory_high,
     ))
     .await?)
 }
@@ -305,6 +311,7 @@ fn resource_limits_request(
     cpu_quota: &str,
     cpu_weight: Option<u32>,
     io_weight: Option<u32>,
+    memory_high: Option<u64>,
 ) -> PrivRequest {
     PrivRequest::WriteResourceLimits {
         container: container.to_owned(),
@@ -312,6 +319,7 @@ fn resource_limits_request(
         cpu_quota: cpu_quota.to_owned(),
         cpu_weight,
         io_weight,
+        memory_high,
     }
 }
 
@@ -654,6 +662,7 @@ mod tests {
             "200%",
             Some(50),
             None,
+            Some(3_865_470_566),
         ));
         assert_eq!(got["op"], "write_resource_limits");
         assert_eq!(got["container"], "alice");
@@ -661,6 +670,7 @@ mod tests {
         assert_eq!(got["cpu_quota"], "200%");
         assert_eq!(got["cpu_weight"], 50);
         assert_eq!(got["io_weight"], Value::Null);
+        assert_eq!(got["memory_high"], 3_865_470_566_u64);
     }
 
     #[test]