egress-pin: a hung pass fails instead of stopping every later pass
The lock wait polls try_lock and gives up at the pass deadline (30 s), nft, ip and nsenter get 10 s each before they are killed, and both pin units get TimeoutStartSec=90s, which systemd does not set for a oneshot unit.
This commit is contained in:
parent
cd64cbae82
commit
1552019580
1 changed files with 221 additions and 31 deletions
|
|
@ -30,10 +30,11 @@
|
|||
//! again on the next pass.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::Write as _;
|
||||
use std::fs::{File, OpenOptions, TryLockError};
|
||||
use std::io::{Read, Write as _};
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Stdio};
|
||||
use std::process::{Command, Output, Stdio};
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context as _, Result, bail};
|
||||
|
|
@ -68,6 +69,16 @@ const WAIT: Duration = Duration::from_secs(30);
|
|||
/// The pause between two looks while waiting.
|
||||
const POLL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// How long `ip`, `nsenter` or `nft` may run before it is killed: a pass runs
|
||||
/// them under the lock of the bindings, and they take milliseconds.
|
||||
const CHILD_LIMIT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// The pause between two looks at a running child.
|
||||
const CHILD_POLL: Duration = Duration::from_millis(10);
|
||||
|
||||
/// How long a finished child's output is waited for on its pipes.
|
||||
const DRAIN: Duration = Duration::from_secs(1);
|
||||
|
||||
/// A lease timeout when dnsmasq names no remaining time.
|
||||
const DEFAULT_LEASE_SECS: u64 = 3600;
|
||||
|
||||
|
|
@ -593,12 +604,78 @@ pub(crate) fn container_leader(state: &str) -> Option<u32> {
|
|||
leader.filter(|_| container)
|
||||
}
|
||||
|
||||
/// Run `program` with `args`, `input` on its stdin, and collect what it
|
||||
/// prints. A child still running after `limit` is killed and the call fails: a
|
||||
/// pass holds the lock of the bindings while it runs `ip`, `nsenter` and
|
||||
/// `nft`, and one that never returns would stop every later pass.
|
||||
fn run_bounded(
|
||||
program: &str,
|
||||
args: &[&str],
|
||||
input: Option<&str>,
|
||||
limit: Duration,
|
||||
) -> Result<Output> {
|
||||
let mut child = Command::new(program)
|
||||
.args(args)
|
||||
.stdin(if input.is_some() {
|
||||
Stdio::piped()
|
||||
} else {
|
||||
Stdio::null()
|
||||
})
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("run {program}"))?;
|
||||
// The pipes are read and written on threads: a child that stops reading its
|
||||
// input, or fills an output pipe nobody empties, must not hold this one.
|
||||
if let (Some(text), Some(mut pipe)) = (input, child.stdin.take()) {
|
||||
let text = text.to_string();
|
||||
std::thread::spawn(move || pipe.write_all(text.as_bytes()));
|
||||
}
|
||||
let stdout = drain(child.stdout.take());
|
||||
let stderr = drain(child.stderr.take());
|
||||
let deadline = Instant::now() + limit;
|
||||
let status = loop {
|
||||
if let Some(status) = child
|
||||
.try_wait()
|
||||
.with_context(|| format!("wait for {program}"))?
|
||||
{
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
// Reaped on a thread: a child stuck in the kernel can outlast the
|
||||
// kill, and this pass must not wait for it.
|
||||
std::thread::spawn(move || child.wait());
|
||||
bail!(
|
||||
"{program} {} did not finish in {limit:?} and was killed",
|
||||
args.join(" ")
|
||||
);
|
||||
}
|
||||
std::thread::sleep(CHILD_POLL);
|
||||
};
|
||||
Ok(Output {
|
||||
status,
|
||||
stdout: stdout.recv_timeout(DRAIN).unwrap_or_default(),
|
||||
stderr: stderr.recv_timeout(DRAIN).unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Read `pipe` to its end on a thread; the bytes arrive on the channel.
|
||||
fn drain(pipe: Option<impl Read + Send + 'static>) -> mpsc::Receiver<Vec<u8>> {
|
||||
let (send, receive) = mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut bytes = Vec::new();
|
||||
if let Some(mut pipe) = pipe {
|
||||
let _ = pipe.read_to_end(&mut bytes);
|
||||
}
|
||||
let _ = send.send(bytes);
|
||||
});
|
||||
receive
|
||||
}
|
||||
|
||||
/// Run `program` with `args` and return its stdout.
|
||||
fn output_of(program: &str, args: &[&str]) -> Result<String> {
|
||||
let out = Command::new(program)
|
||||
.args(args)
|
||||
.output()
|
||||
.with_context(|| format!("run {program}"))?;
|
||||
let out = run_bounded(program, args, None, CHILD_LIMIT)?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"{program} {} exited {}: {}",
|
||||
|
|
@ -612,19 +689,7 @@ fn output_of(program: &str, args: &[&str]) -> Result<String> {
|
|||
|
||||
/// Feed `script` to `nft -f -`.
|
||||
fn nft(script: &str) -> Result<()> {
|
||||
let mut child = Command::new("nft")
|
||||
.args(["-f", "-"])
|
||||
.stdin(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("run nft")?;
|
||||
child
|
||||
.stdin
|
||||
.take()
|
||||
.context("nft stdin")?
|
||||
.write_all(script.as_bytes())
|
||||
.context("write to nft")?;
|
||||
let out = child.wait_with_output().context("wait for nft")?;
|
||||
let out = run_bounded("nft", &["-f", "-"], Some(script), CHILD_LIMIT)?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"nft exited {}: {}",
|
||||
|
|
@ -728,17 +793,39 @@ fn commit(survey: &Survey) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for the lock of the bindings. It is released when the file is dropped.
|
||||
fn lock() -> Result<File> {
|
||||
/// Wait for the lock of the bindings until `deadline`. It is released when the
|
||||
/// file is dropped.
|
||||
fn lock(deadline: Instant) -> Result<File> {
|
||||
let path = std::env::var("HIVE_EGRESS_LOCK").unwrap_or_else(|_| LOCK.into());
|
||||
lock_at(&path, deadline)
|
||||
}
|
||||
|
||||
/// Take the flock of `path`, polling while another pass holds it. A pass that
|
||||
/// still waits at `deadline` gives up, so a wedged holder does not stop every
|
||||
/// later pass: the unit fails and the next start tries again.
|
||||
fn lock_at(path: &str, deadline: Instant) -> Result<File> {
|
||||
let started = Instant::now();
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(&path)
|
||||
.open(path)
|
||||
.with_context(|| format!("open {path}"))?;
|
||||
file.lock().with_context(|| format!("lock {path}"))?;
|
||||
Ok(file)
|
||||
loop {
|
||||
match file.try_lock() {
|
||||
Ok(()) => return Ok(file),
|
||||
Err(TryLockError::WouldBlock) if Instant::now() < deadline => {
|
||||
std::thread::sleep(POLL);
|
||||
}
|
||||
Err(TryLockError::WouldBlock) => bail!(
|
||||
"lock held by another pass for {} s; giving up",
|
||||
started.elapsed().as_secs()
|
||||
),
|
||||
Err(TryLockError::Error(e)) => {
|
||||
return Err(e).with_context(|| format!("lock {path}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
|
|
@ -781,7 +868,7 @@ fn pin(args: &[String]) -> Result<()> {
|
|||
let deadline = Instant::now() + WAIT;
|
||||
loop {
|
||||
let survey = {
|
||||
let _lock = lock()?;
|
||||
let _lock = lock(deadline)?;
|
||||
let survey = survey(&bridge)?;
|
||||
nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?;
|
||||
commit(&survey)?;
|
||||
|
|
@ -807,9 +894,7 @@ fn pin(args: &[String]) -> Result<()> {
|
|||
/// Whether an interface of this name exists, on any bridge or none: a port is
|
||||
/// created before it joins the bridge.
|
||||
fn port_exists(name: &str) -> bool {
|
||||
Command::new("ip")
|
||||
.args(["link", "show", "dev", name])
|
||||
.output()
|
||||
run_bounded("ip", &["link", "show", "dev", name], None, CHILD_LIMIT)
|
||||
.is_ok_and(|o| o.status.success())
|
||||
}
|
||||
|
||||
|
|
@ -846,7 +931,7 @@ fn load(args: &[String]) -> Result<()> {
|
|||
&std::fs::read_to_string(&path).unwrap_or_default(),
|
||||
now_secs(),
|
||||
);
|
||||
let _lock = lock()?;
|
||||
let _lock = lock(Instant::now() + WAIT)?;
|
||||
let found = survey(&bridge);
|
||||
let pins = match &found {
|
||||
Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()),
|
||||
|
|
@ -1481,6 +1566,88 @@ mod tests {
|
|||
assert_eq!(container_leader("CLASS=container\n"), None);
|
||||
}
|
||||
|
||||
fn lock_path(name: &str) -> String {
|
||||
let path =
|
||||
std::env::temp_dir().join(format!("hive-priv-lock-{}-{name}", std::process::id()));
|
||||
path.to_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lock_held_past_the_deadline_is_an_error_and_a_free_one_is_taken() {
|
||||
let path = lock_path("held");
|
||||
let held = lock_at(&path, Instant::now()).unwrap();
|
||||
let started = Instant::now();
|
||||
let err = lock_at(&path, started + Duration::from_millis(300)).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("lock held by another pass"),
|
||||
"{err:#}"
|
||||
);
|
||||
assert!(started.elapsed() >= Duration::from_millis(300));
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
drop(held);
|
||||
lock_at(&path, Instant::now()).unwrap();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lock_released_while_another_pass_waits_is_taken_by_it() {
|
||||
let path = lock_path("released");
|
||||
let held = lock_at(&path, Instant::now()).unwrap();
|
||||
let releaser = std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
drop(held);
|
||||
});
|
||||
let started = Instant::now();
|
||||
lock_at(&path, started + Duration::from_secs(10)).unwrap();
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
releaser.join().unwrap();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_child_that_outlives_its_limit_is_killed_and_the_call_fails() {
|
||||
let started = Instant::now();
|
||||
let err = run_bounded("sleep", &["30"], None, Duration::from_millis(200)).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("sleep 30 did not finish in"),
|
||||
"{err:#}"
|
||||
);
|
||||
assert!(started.elapsed() < Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_child_that_outlives_its_limit_is_gone_afterwards() {
|
||||
let pid_file = lock_path("pid");
|
||||
let script = format!("echo $$ > {pid_file}; exec sleep 30");
|
||||
let _ = run_bounded("sh", &["-c", &script], None, Duration::from_millis(500)).unwrap_err();
|
||||
let pid = std::fs::read_to_string(&pid_file)
|
||||
.unwrap()
|
||||
.trim()
|
||||
.to_string();
|
||||
let _ = std::fs::remove_file(&pid_file);
|
||||
let proc = format!("/proc/{pid}");
|
||||
let gone = (0..60).any(|_| {
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
!Path::new(&proc).exists()
|
||||
});
|
||||
assert!(gone, "{proc} is still there");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_child_that_finishes_gives_its_output_and_status_and_takes_its_input() {
|
||||
let limit = Duration::from_secs(10);
|
||||
let out = run_bounded("cat", &[], Some("hello"), limit).unwrap();
|
||||
assert!(out.status.success());
|
||||
assert_eq!(out.stdout, b"hello");
|
||||
let out = run_bounded("sh", &["-c", "echo oops >&2; exit 3"], None, limit).unwrap();
|
||||
assert_eq!(out.status.code(), Some(3));
|
||||
assert_eq!(out.stderr, b"oops\n");
|
||||
// More than a pipe holds: a child that blocks on a full pipe would
|
||||
// never finish.
|
||||
let out = run_bounded("head", &["-c", "300000", "/dev/zero"], None, limit).unwrap();
|
||||
assert_eq!(out.stdout.len(), 300_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_names_only() {
|
||||
assert!(is_plain_name("vb-h-atlas"));
|
||||
|
|
|
|||
Loading…
Reference in a new issue