Watch
0
0
Fork
You've already forked hyperhive
0

egress-pin: a hung pass fails instead of stopping every later pass

The lock wait polls try_lock and gives up at the pass deadline (30 s), nft, ip
and nsenter get 10 s each before they are killed, and both pin units get
TimeoutStartSec=90s, which systemd does not set for a oneshot unit.
This commit is contained in:
maia 2026-11-19 20:03:14 +01:00 • committed by mara
commit 1552019580

View file

@ -30,10 +30,11 @@
//! again on the next pass.
use std::fmt::Write as _;
use std::fs::{File, OpenOptions};
use std::io::Write as _;
use std::fs::{File, OpenOptions, TryLockError};
use std::io::{Read, Write as _};
use std::path::Path;
use std::process::{Command, Stdio};
use std::process::{Command, Output, Stdio};
use std::sync::mpsc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context as _, Result, bail};
@ -68,6 +69,16 @@ const WAIT: Duration = Duration::from_secs(30);
/// The pause between two looks while waiting.
const POLL: Duration = Duration::from_millis(250);
/// How long `ip`, `nsenter` or `nft` may run before it is killed: a pass runs
/// them under the lock of the bindings, and they take milliseconds.
const CHILD_LIMIT: Duration = Duration::from_secs(10);
/// The pause between two looks at a running child.
const CHILD_POLL: Duration = Duration::from_millis(10);
/// How long a finished child's output is waited for on its pipes.
const DRAIN: Duration = Duration::from_secs(1);
/// A lease timeout when dnsmasq names no remaining time.
const DEFAULT_LEASE_SECS: u64 = 3600;
@ -593,12 +604,78 @@ pub(crate) fn container_leader(state: &str) -> Option<u32> {
leader.filter(|_| container)
}
/// Run `program` with `args`, `input` on its stdin, and collect what it
/// prints. A child still running after `limit` is killed and the call fails: a
/// pass holds the lock of the bindings while it runs `ip`, `nsenter` and
/// `nft`, and one that never returns would stop every later pass.
fn run_bounded(
program: &str,
args: &[&str],
input: Option<&str>,
limit: Duration,
) -> Result<Output> {
let mut child = Command::new(program)
.args(args)
.stdin(if input.is_some() {
Stdio::piped()
} else {
Stdio::null()
})
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.with_context(|| format!("run {program}"))?;
// The pipes are read and written on threads: a child that stops reading its
// input, or fills an output pipe nobody empties, must not hold this one.
if let (Some(text), Some(mut pipe)) = (input, child.stdin.take()) {
let text = text.to_string();
std::thread::spawn(move || pipe.write_all(text.as_bytes()));
}
let stdout = drain(child.stdout.take());
let stderr = drain(child.stderr.take());
let deadline = Instant::now() + limit;
let status = loop {
if let Some(status) = child
.try_wait()
.with_context(|| format!("wait for {program}"))?
{
break status;
}
if Instant::now() >= deadline {
let _ = child.kill();
// Reaped on a thread: a child stuck in the kernel can outlast the
// kill, and this pass must not wait for it.
std::thread::spawn(move || child.wait());
bail!(
"{program} {} did not finish in {limit:?} and was killed",
args.join(" ")
);
}
std::thread::sleep(CHILD_POLL);
};
Ok(Output {
status,
stdout: stdout.recv_timeout(DRAIN).unwrap_or_default(),
stderr: stderr.recv_timeout(DRAIN).unwrap_or_default(),
})
}
/// Read `pipe` to its end on a thread; the bytes arrive on the channel.
fn drain(pipe: Option<impl Read + Send + 'static>) -> mpsc::Receiver<Vec<u8>> {
let (send, receive) = mpsc::channel();
std::thread::spawn(move || {
let mut bytes = Vec::new();
if let Some(mut pipe) = pipe {
let _ = pipe.read_to_end(&mut bytes);
}
let _ = send.send(bytes);
});
receive
}
/// Run `program` with `args` and return its stdout.
fn output_of(program: &str, args: &[&str]) -> Result<String> {
let out = Command::new(program)
.args(args)
.output()
.with_context(|| format!("run {program}"))?;
let out = run_bounded(program, args, None, CHILD_LIMIT)?;
if !out.status.success() {
bail!(
"{program} {} exited {}: {}",
@ -612,19 +689,7 @@ fn output_of(program: &str, args: &[&str]) -> Result<String> {
/// Feed `script` to `nft -f -`.
fn nft(script: &str) -> Result<()> {
let mut child = Command::new("nft")
.args(["-f", "-"])
.stdin(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("run nft")?;
child
.stdin
.take()
.context("nft stdin")?
.write_all(script.as_bytes())
.context("write to nft")?;
let out = child.wait_with_output().context("wait for nft")?;
let out = run_bounded("nft", &["-f", "-"], Some(script), CHILD_LIMIT)?;
if !out.status.success() {
bail!(
"nft exited {}: {}",
@ -728,17 +793,39 @@ fn commit(survey: &Survey) -> Result<()> {
Ok(())
}
/// Wait for the lock of the bindings. It is released when the file is dropped.
fn lock() -> Result<File> {
/// Wait for the lock of the bindings until `deadline`. It is released when the
/// file is dropped.
fn lock(deadline: Instant) -> Result<File> {
let path = std::env::var("HIVE_EGRESS_LOCK").unwrap_or_else(|_| LOCK.into());
lock_at(&path, deadline)
}
/// Take the flock of `path`, polling while another pass holds it. A pass that
/// still waits at `deadline` gives up, so a wedged holder does not stop every
/// later pass: the unit fails and the next start tries again.
fn lock_at(path: &str, deadline: Instant) -> Result<File> {
let started = Instant::now();
let file = OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.open(&path)
.open(path)
.with_context(|| format!("open {path}"))?;
file.lock().with_context(|| format!("lock {path}"))?;
Ok(file)
loop {
match file.try_lock() {
Ok(()) => return Ok(file),
Err(TryLockError::WouldBlock) if Instant::now() < deadline => {
std::thread::sleep(POLL);
}
Err(TryLockError::WouldBlock) => bail!(
"lock held by another pass for {} s; giving up",
started.elapsed().as_secs()
),
Err(TryLockError::Error(e)) => {
return Err(e).with_context(|| format!("lock {path}"));
}
}
}
}
fn now_secs() -> u64 {
@ -781,7 +868,7 @@ fn pin(args: &[String]) -> Result<()> {
let deadline = Instant::now() + WAIT;
loop {
let survey = {
let _lock = lock()?;
let _lock = lock(deadline)?;
let survey = survey(&bridge)?;
nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?;
commit(&survey)?;
@ -807,9 +894,7 @@ fn pin(args: &[String]) -> Result<()> {
/// Whether an interface of this name exists, on any bridge or none: a port is
/// created before it joins the bridge.
fn port_exists(name: &str) -> bool {
Command::new("ip")
.args(["link", "show", "dev", name])
.output()
run_bounded("ip", &["link", "show", "dev", name], None, CHILD_LIMIT)
.is_ok_and(|o| o.status.success())
}
@ -846,7 +931,7 @@ fn load(args: &[String]) -> Result<()> {
&std::fs::read_to_string(&path).unwrap_or_default(),
now_secs(),
);
let _lock = lock()?;
let _lock = lock(Instant::now() + WAIT)?;
let found = survey(&bridge);
let pins = match &found {
Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()),
@ -1481,6 +1566,88 @@ mod tests {
assert_eq!(container_leader("CLASS=container\n"), None);
}
fn lock_path(name: &str) -> String {
let path =
std::env::temp_dir().join(format!("hive-priv-lock-{}-{name}", std::process::id()));
path.to_str().unwrap().to_string()
}
#[test]
fn a_lock_held_past_the_deadline_is_an_error_and_a_free_one_is_taken() {
let path = lock_path("held");
let held = lock_at(&path, Instant::now()).unwrap();
let started = Instant::now();
let err = lock_at(&path, started + Duration::from_millis(300)).unwrap_err();
assert!(
err.to_string().contains("lock held by another pass"),
"{err:#}"
);
assert!(started.elapsed() >= Duration::from_millis(300));
assert!(started.elapsed() < Duration::from_secs(5));
drop(held);
lock_at(&path, Instant::now()).unwrap();
let _ = std::fs::remove_file(&path);
}
#[test]
fn a_lock_released_while_another_pass_waits_is_taken_by_it() {
let path = lock_path("released");
let held = lock_at(&path, Instant::now()).unwrap();
let releaser = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(300));
drop(held);
});
let started = Instant::now();
lock_at(&path, started + Duration::from_secs(10)).unwrap();
assert!(started.elapsed() < Duration::from_secs(5));
releaser.join().unwrap();
let _ = std::fs::remove_file(&path);
}
#[test]
fn a_child_that_outlives_its_limit_is_killed_and_the_call_fails() {
let started = Instant::now();
let err = run_bounded("sleep", &["30"], None, Duration::from_millis(200)).unwrap_err();
assert!(
err.to_string().contains("sleep 30 did not finish in"),
"{err:#}"
);
assert!(started.elapsed() < Duration::from_secs(5));
}
#[test]
fn a_child_that_outlives_its_limit_is_gone_afterwards() {
let pid_file = lock_path("pid");
let script = format!("echo $$ > {pid_file}; exec sleep 30");
let _ = run_bounded("sh", &["-c", &script], None, Duration::from_millis(500)).unwrap_err();
let pid = std::fs::read_to_string(&pid_file)
.unwrap()
.trim()
.to_string();
let _ = std::fs::remove_file(&pid_file);
let proc = format!("/proc/{pid}");
let gone = (0..60).any(|_| {
std::thread::sleep(Duration::from_millis(50));
!Path::new(&proc).exists()
});
assert!(gone, "{proc} is still there");
}
#[test]
fn a_child_that_finishes_gives_its_output_and_status_and_takes_its_input() {
let limit = Duration::from_secs(10);
let out = run_bounded("cat", &[], Some("hello"), limit).unwrap();
assert!(out.status.success());
assert_eq!(out.stdout, b"hello");
let out = run_bounded("sh", &["-c", "echo oops >&2; exit 3"], None, limit).unwrap();
assert_eq!(out.status.code(), Some(3));
assert_eq!(out.stderr, b"oops\n");
// More than a pipe holds: a child that blocks on a full pipe would
// never finish.
let out = run_bounded("head", &["-c", "300000", "/dev/zero"], None, limit).unwrap();
assert_eq!(out.stdout.len(), 300_000);
}
#[test]
fn plain_names_only() {
assert!(is_plain_name("vb-h-atlas"));