diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 697df0978..2ff58f4fb 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -826,7 +826,14 @@ source check lives in that table and the host-side filtering stays in for it. udev starts it for each new `vb-*` port (`hive-egress-pin@.service`), and a timer (`hive-egress-pin.timer`, every 30 seconds) runs it for the ports that still have no binding. The timer reads - no container while every port has one. The `hive-egress` unit loads the tables + no container while every port has one. One pass at a time reads and writes the + bindings, under a lock. A pass waits 30 seconds for the lock and gives up with + `lock held by another pass for 30 s; giving up`, and `ip`, `nsenter` and `nft` + each get 10 seconds before the pass kills them and fails. Both units also have + `TimeoutStartSec=90s`. A pass that stops responding fails visibly and the + next run of the timer or udev starts again, instead of stopping every later + pass. The + `hive-egress` unit loads the tables with `hive-priv egress-load`: it looks up the bindings first, while the old tables still filter, and puts them and the leases from dnsmasq's lease file into the same nft transaction as the tables. A start or reload never leaves diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 1158068b9..1b94cec9d 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -30,10 +30,11 @@ //! again on the next pass. use std::fmt::Write as _; -use std::fs::{File, OpenOptions}; -use std::io::Write as _; +use std::fs::{File, OpenOptions, TryLockError}; +use std::io::{Read, Write as _}; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Command, Output, Stdio}; +use std::sync::mpsc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::{Context as _, Result, bail}; @@ -68,6 +69,16 @@ const WAIT: Duration = Duration::from_secs(30); /// The pause between two looks while waiting. const POLL: Duration = Duration::from_millis(250); +/// How long `ip`, `nsenter` or `nft` may run before it is killed: a pass runs +/// them under the lock of the bindings, and they take milliseconds. +const CHILD_LIMIT: Duration = Duration::from_secs(10); + +/// The pause between two looks at a running child. +const CHILD_POLL: Duration = Duration::from_millis(10); + +/// How long a finished child's output is waited for on its pipes. +const DRAIN: Duration = Duration::from_secs(1); + /// A lease timeout when dnsmasq names no remaining time. const DEFAULT_LEASE_SECS: u64 = 3600; @@ -593,12 +604,78 @@ pub(crate) fn container_leader(state: &str) -> Option { leader.filter(|_| container) } +/// Run `program` with `args`, `input` on its stdin, and collect what it +/// prints. A child still running after `limit` is killed and the call fails: a +/// pass holds the lock of the bindings while it runs `ip`, `nsenter` and +/// `nft`, and one that never returns would stop every later pass. +fn run_bounded( + program: &str, + args: &[&str], + input: Option<&str>, + limit: Duration, +) -> Result { + let mut child = Command::new(program) + .args(args) + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("run {program}"))?; + // The pipes are read and written on threads: a child that stops reading its + // input, or fills an output pipe nobody empties, must not hold this one. + if let (Some(text), Some(mut pipe)) = (input, child.stdin.take()) { + let text = text.to_string(); + std::thread::spawn(move || pipe.write_all(text.as_bytes())); + } + let stdout = drain(child.stdout.take()); + let stderr = drain(child.stderr.take()); + let deadline = Instant::now() + limit; + let status = loop { + if let Some(status) = child + .try_wait() + .with_context(|| format!("wait for {program}"))? + { + break status; + } + if Instant::now() >= deadline { + let _ = child.kill(); + // Reaped on a thread: a child stuck in the kernel can outlast the + // kill, and this pass must not wait for it. + std::thread::spawn(move || child.wait()); + bail!( + "{program} {} did not finish in {limit:?} and was killed", + args.join(" ") + ); + } + std::thread::sleep(CHILD_POLL); + }; + Ok(Output { + status, + stdout: stdout.recv_timeout(DRAIN).unwrap_or_default(), + stderr: stderr.recv_timeout(DRAIN).unwrap_or_default(), + }) +} + +/// Read `pipe` to its end on a thread; the bytes arrive on the channel. +fn drain(pipe: Option) -> mpsc::Receiver> { + let (send, receive) = mpsc::channel(); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_end(&mut bytes); + } + let _ = send.send(bytes); + }); + receive +} + /// Run `program` with `args` and return its stdout. fn output_of(program: &str, args: &[&str]) -> Result { - let out = Command::new(program) - .args(args) - .output() - .with_context(|| format!("run {program}"))?; + let out = run_bounded(program, args, None, CHILD_LIMIT)?; if !out.status.success() { bail!( "{program} {} exited {}: {}", @@ -612,19 +689,7 @@ fn output_of(program: &str, args: &[&str]) -> Result { /// Feed `script` to `nft -f -`. fn nft(script: &str) -> Result<()> { - let mut child = Command::new("nft") - .args(["-f", "-"]) - .stdin(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("run nft")?; - child - .stdin - .take() - .context("nft stdin")? - .write_all(script.as_bytes()) - .context("write to nft")?; - let out = child.wait_with_output().context("wait for nft")?; + let out = run_bounded("nft", &["-f", "-"], Some(script), CHILD_LIMIT)?; if !out.status.success() { bail!( "nft exited {}: {}", @@ -728,17 +793,39 @@ fn commit(survey: &Survey) -> Result<()> { Ok(()) } -/// Wait for the lock of the bindings. It is released when the file is dropped. -fn lock() -> Result { +/// Wait for the lock of the bindings until `deadline`. It is released when the +/// file is dropped. +fn lock(deadline: Instant) -> Result { let path = std::env::var("HIVE_EGRESS_LOCK").unwrap_or_else(|_| LOCK.into()); + lock_at(&path, deadline) +} + +/// Take the flock of `path`, polling while another pass holds it. A pass that +/// still waits at `deadline` gives up, so a wedged holder does not stop every +/// later pass: the unit fails and the next start tries again. +fn lock_at(path: &str, deadline: Instant) -> Result { + let started = Instant::now(); let file = OpenOptions::new() .write(true) .create(true) .truncate(false) - .open(&path) + .open(path) .with_context(|| format!("open {path}"))?; - file.lock().with_context(|| format!("lock {path}"))?; - Ok(file) + loop { + match file.try_lock() { + Ok(()) => return Ok(file), + Err(TryLockError::WouldBlock) if Instant::now() < deadline => { + std::thread::sleep(POLL); + } + Err(TryLockError::WouldBlock) => bail!( + "lock held by another pass for {} s; giving up", + started.elapsed().as_secs() + ), + Err(TryLockError::Error(e)) => { + return Err(e).with_context(|| format!("lock {path}")); + } + } + } } fn now_secs() -> u64 { @@ -781,7 +868,7 @@ fn pin(args: &[String]) -> Result<()> { let deadline = Instant::now() + WAIT; loop { let survey = { - let _lock = lock()?; + let _lock = lock(deadline)?; let survey = survey(&bridge)?; nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?; commit(&survey)?; @@ -807,9 +894,7 @@ fn pin(args: &[String]) -> Result<()> { /// Whether an interface of this name exists, on any bridge or none: a port is /// created before it joins the bridge. fn port_exists(name: &str) -> bool { - Command::new("ip") - .args(["link", "show", "dev", name]) - .output() + run_bounded("ip", &["link", "show", "dev", name], None, CHILD_LIMIT) .is_ok_and(|o| o.status.success()) } @@ -846,7 +931,7 @@ fn load(args: &[String]) -> Result<()> { &std::fs::read_to_string(&path).unwrap_or_default(), now_secs(), ); - let _lock = lock()?; + let _lock = lock(Instant::now() + WAIT)?; let found = survey(&bridge); let pins = match &found { Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()), @@ -1481,6 +1566,88 @@ mod tests { assert_eq!(container_leader("CLASS=container\n"), None); } + fn lock_path(name: &str) -> String { + let path = + std::env::temp_dir().join(format!("hive-priv-lock-{}-{name}", std::process::id())); + path.to_str().unwrap().to_string() + } + + #[test] + fn a_lock_held_past_the_deadline_is_an_error_and_a_free_one_is_taken() { + let path = lock_path("held"); + let held = lock_at(&path, Instant::now()).unwrap(); + let started = Instant::now(); + let err = lock_at(&path, started + Duration::from_millis(300)).unwrap_err(); + assert!( + err.to_string().contains("lock held by another pass"), + "{err:#}" + ); + assert!(started.elapsed() >= Duration::from_millis(300)); + assert!(started.elapsed() < Duration::from_secs(5)); + drop(held); + lock_at(&path, Instant::now()).unwrap(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn a_lock_released_while_another_pass_waits_is_taken_by_it() { + let path = lock_path("released"); + let held = lock_at(&path, Instant::now()).unwrap(); + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(300)); + drop(held); + }); + let started = Instant::now(); + lock_at(&path, started + Duration::from_secs(10)).unwrap(); + assert!(started.elapsed() < Duration::from_secs(5)); + releaser.join().unwrap(); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn a_child_that_outlives_its_limit_is_killed_and_the_call_fails() { + let started = Instant::now(); + let err = run_bounded("sleep", &["30"], None, Duration::from_millis(200)).unwrap_err(); + assert!( + err.to_string().contains("sleep 30 did not finish in"), + "{err:#}" + ); + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_child_that_outlives_its_limit_is_gone_afterwards() { + let pid_file = lock_path("pid"); + let script = format!("echo $$ > {pid_file}; exec sleep 30"); + let _ = run_bounded("sh", &["-c", &script], None, Duration::from_millis(500)).unwrap_err(); + let pid = std::fs::read_to_string(&pid_file) + .unwrap() + .trim() + .to_string(); + let _ = std::fs::remove_file(&pid_file); + let proc = format!("/proc/{pid}"); + let gone = (0..60).any(|_| { + std::thread::sleep(Duration::from_millis(50)); + !Path::new(&proc).exists() + }); + assert!(gone, "{proc} is still there"); + } + + #[test] + fn a_child_that_finishes_gives_its_output_and_status_and_takes_its_input() { + let limit = Duration::from_secs(10); + let out = run_bounded("cat", &[], Some("hello"), limit).unwrap(); + assert!(out.status.success()); + assert_eq!(out.stdout, b"hello"); + let out = run_bounded("sh", &["-c", "echo oops >&2; exit 3"], None, limit).unwrap(); + assert_eq!(out.status.code(), Some(3)); + assert_eq!(out.stderr, b"oops\n"); + // More than a pipe holds: a child that blocks on a full pipe would + // never finish. + let out = run_bounded("head", &["-c", "300000", "/dev/zero"], None, limit).unwrap(); + assert_eq!(out.stdout.len(), 300_000); + } + #[test] fn plain_names_only() { assert!(is_plain_name("vb-h-atlas")); diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index e2a51b49c..960f098b3 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -64,6 +64,11 @@ let pkgs.iproute2 pkgs.util-linux ]; + # systemd gives a oneshot unit no start timeout unless it is set. A pass takes + # the lock for up to 30 s and runs each child for up to 10 s (hive-priv's + # `egress.rs`); this is above both, and kills a pass that still hangs so the + # timer and udev can start the next. + pinTimeout = "90s"; pinEnvironment = { HIVE_EGRESS_BRIDGE = br; } @@ -619,6 +624,7 @@ in serviceConfig = { Type = "oneshot"; ExecStart = pinCommand "%i"; + TimeoutStartSec = pinTimeout; }; }; @@ -634,6 +640,7 @@ in serviceConfig = { Type = "oneshot"; ExecStart = pinCommand ""; + TimeoutStartSec = pinTimeout; }; }; systemd.timers.hive-egress-pin = { diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 0ff868188..8f740db45 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1756,6 +1756,15 @@ let && !(centralToggleOff.systemd.services ? "hive-egress-pin@") && !(centralToggleOff.systemd ? timers && centralToggleOff.systemd.timers ? hive-egress-pin); } + { + # systemd gives a oneshot unit no start timeout unless one is set, and a + # pass that hangs holds the lock every later pass waits for. 90 s is above + # the 30 s a pass waits for the lock plus the 10 s each child may run. + name = "a hung egress-pin pass is killed after 90 seconds, in the per-port unit and the sweeper alike"; + ok = + bare.systemd.services."hive-egress-pin@".serviceConfig.TimeoutStartSec == "90s" + && bare.systemd.services.hive-egress-pin.serviceConfig.TimeoutStartSec == "90s"; + } { # `accept` renders the table without the forward chain and without the # log prefix; the input chain is the presence arm that the text it