bash tasks are the OOM killer's first victims: score 1000, subagent runs 900, MemoryHigh on the bash daemon
This commit is contained in:
parent
d4dc8b48b1
commit
77f33a23d8
1 changed files with 155 additions and 32 deletions
|
|
@ -41,6 +41,16 @@ const ENTRY_PROGRAM_ENV: &str = "HIVE_RUN_ENTRY_PROGRAM";
|
|||
const ENTRY_CGROUP_ENV: &str = "HIVE_RUN_ENTRY_CGROUP";
|
||||
const ENTRY_OOM_SCORE_ADJ_ENV: &str = "HIVE_RUN_ENTRY_OOM_SCORE_ADJ";
|
||||
|
||||
/// A bash task's `oom_score_adj`: the maximum, so under memory pressure the
|
||||
/// kernel takes a bash task first. A runaway `cargo test` or `nix build` is a
|
||||
/// task, and the subagent runs and the agent's own session are worth more.
|
||||
pub const BASH_TASK_OOM_SCORE_ADJ: i32 = 1000;
|
||||
|
||||
/// A subagent run's `oom_score_adj`: below a bash task, above everything else, so
|
||||
/// the order is bash task, then subagent run, then the agent's own session (-500,
|
||||
/// `nix/agent-modules/agent-service.nix`).
|
||||
pub const SUBAGENT_RUN_OOM_SCORE_ADJ: i32 = 900;
|
||||
|
||||
/// How long a run's cgroup may stay populated after the run ended before
|
||||
/// removal is left to the next daemon start.
|
||||
const RELEASE_GRACE: Duration = Duration::from_secs(30);
|
||||
|
|
@ -268,6 +278,50 @@ pub fn enter_in_pre_exec(procs: RawFd) -> io::Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the calling process's `oom_score_adj` to `adj`. Only `open(2)`,
|
||||
/// `write(2)` and `close(2)` on a stack buffer, so it is safe between fork and
|
||||
/// exec. Raising the score needs no privilege.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// The `open(2)` or `write(2)` failure.
|
||||
pub fn set_oom_score_adj_in_pre_exec(adj: i32) -> io::Result<()> {
|
||||
// Digits of `adj`, written backwards: no allocation, no `format!`.
|
||||
let mut buf = [0_u8; 12];
|
||||
let mut at = buf.len();
|
||||
let mut n = adj.unsigned_abs();
|
||||
loop {
|
||||
at -= 1;
|
||||
buf[at] = b'0' + u8::try_from(n % 10).unwrap_or(0);
|
||||
n /= 10;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if adj < 0 {
|
||||
at -= 1;
|
||||
buf[at] = b'-';
|
||||
}
|
||||
// SAFETY: a NUL-terminated static path, then a write of a stack buffer to
|
||||
// the fd just opened, which is closed again.
|
||||
unsafe {
|
||||
let fd = libc::open(
|
||||
c"/proc/self/oom_score_adj".as_ptr(),
|
||||
libc::O_WRONLY | libc::O_CLOEXEC,
|
||||
);
|
||||
if fd < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let wrote = libc::write(fd, buf[at..].as_ptr().cast(), buf.len() - at);
|
||||
let err = io::Error::last_os_error();
|
||||
libc::close(fd);
|
||||
if wrote < 0 {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// When this process was started by [`entry`]: enter the run's
|
||||
/// cgroup if it has one, set its `oom_score_adj`, and exec the run's program with this
|
||||
/// process's arguments; on failure print the reason and exit 127. Otherwise
|
||||
|
|
@ -423,6 +477,39 @@ mod tests {
|
|||
assert_eq!(get(ENTRY_CGROUP_ENV), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_kill_order_is_bash_task_then_subagent_run_then_the_session() {
|
||||
// The agent's own session: `OOMScoreAdjust` in
|
||||
// `nix/agent-modules/agent-service.nix`.
|
||||
const SESSION: i32 = -500;
|
||||
assert_eq!(BASH_TASK_OOM_SCORE_ADJ, 1000);
|
||||
assert_eq!(SUBAGENT_RUN_OOM_SCORE_ADJ, 900);
|
||||
assert!(BASH_TASK_OOM_SCORE_ADJ > SUBAGENT_RUN_OOM_SCORE_ADJ);
|
||||
assert!(SUBAGENT_RUN_OOM_SCORE_ADJ > SESSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_oom_score_adj_in_pre_exec_sets_the_score_the_child_reads_back() {
|
||||
use std::process::Command;
|
||||
// 0 is one digit, 7 one, 900 and 1000 are the two scores in use.
|
||||
for adj in [0, 7, SUBAGENT_RUN_OOM_SCORE_ADJ, BASH_TASK_OOM_SCORE_ADJ] {
|
||||
let mut cmd = Command::new("cat");
|
||||
cmd.arg("/proc/self/oom_score_adj");
|
||||
// SAFETY: the hook only calls the function under test, which is
|
||||
// async-signal-safe.
|
||||
unsafe {
|
||||
cmd.pre_exec(move || set_oom_score_adj_in_pre_exec(adj));
|
||||
}
|
||||
let out = cmd.output().expect("cat runs");
|
||||
assert!(out.status.success(), "{out:?}");
|
||||
assert_eq!(
|
||||
String::from_utf8_lossy(&out.stdout).trim(),
|
||||
adj.to_string(),
|
||||
"adj {adj}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enter_in_pre_exec_writes_zero_to_the_fd() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
|
|
|
|||
Loading…
Reference in a new issue