Watch
0
0
Fork
You've already forked hyperhive
0

hive-egress: the lease hook's lines reach the journal at info under egress-lease

dnsmasq relays the hook's stderr at LOG_DEBUG as dnsmasq-script, and the
field log shipper's journald receiver keeps info and above, so no lease
add or renewal was visible in the field. The hook now also writes each
line to /dev/log at info priority with identifier egress-lease. The
stderr print stays. The VM test reads the lines back with
journalctl -t egress-lease -p info.
This commit is contained in:
maia 2026-11-19 15:28:57 +01:00 • committed by mara
commit d9551415bf
3 changed files with 122 additions and 9 deletions

View file

@ -857,9 +857,15 @@ source check lives in that table and the host-side filtering stays in
`DNSMASQ_LEASE_EXPIRES` lies in the past does nothing, and the pair leaves the `DNSMASQ_LEASE_EXPIRES` lies in the past does nothing, and the pair leaves the
table at its own timeout, as the container's lease ends. A `del` for a lease table at its own timeout, as the container's lease ends. A `del` for a lease
that hasn't expired (a release) deletes the pair at once, as does one without that hasn't expired (a release) deletes the pair at once, as does one without
a readable expiry. `egress-lease` logs each `add`, `old` and `del` to dnsmasq's a readable expiry. `egress-lease` logs each `add`, `old` and `del` to the
log with the prefix `egress-lease:`, the MAC, address, port, expiry, and for journal under the identifier `egress-lease`, at info priority, with the prefix
`del` whether it was `applied` or `skipped (stale)`. `egress-lease:`, the MAC, address, port, expiry, and for `del` whether it was
`applied` or `skipped (stale)`. It also prints the line on stderr, which
dnsmasq logs at debug priority as `dnsmasq-script`. The log shipper reads info
and above, so search the store for the `egress-lease` identifier, not for the
dnsmasq copy. The journal send waits at most a second for room in journald's
queue, because dnsmasq runs its lease scripts one at a time. A send that
fails or times out costs that one journal line and nothing else.
- **The window.** The binding takes the MAC the container's link has when the - **The window.** The binding takes the MAC the container's link has when the
command first looks, normally within a second of the veth appearing. Until command first looks, normally within a second of the veth appearing. Until
then root in the container could change its own MAC, but in that time only then root in the container could change its own MAC, but in that time only

View file

@ -32,6 +32,7 @@
use std::fmt::Write as _; use std::fmt::Write as _;
use std::fs::{File, OpenOptions, TryLockError}; use std::fs::{File, OpenOptions, TryLockError};
use std::io::{Read, Write as _}; use std::io::{Read, Write as _};
use std::os::unix::net::UnixDatagram;
use std::path::Path; use std::path::Path;
use std::process::{Command, Output, Stdio}; use std::process::{Command, Output, Stdio};
use std::sync::mpsc; use std::sync::mpsc;
@ -501,6 +502,35 @@ pub(crate) fn lease_log_line(event: &LeaseEvent<'_>, port: Option<&str>) -> Stri
line line
} }
/// The socket journald listens on for syslog datagrams.
const SYSLOG_SOCKET: &str = "/dev/log";
/// The syslog priority of an `info` line from the `daemon` facility: facility
/// 3 × 8 plus severity 6.
const SYSLOG_DAEMON_INFO: u8 = 30;
/// One syslog datagram, `<priority>tag[pid]: line`. journald files it under
/// `SYSLOG_IDENTIFIER=tag` at the datagram's priority.
pub(crate) fn syslog_datagram(tag: &str, pid: u32, line: &str) -> String {
format!("<{SYSLOG_DAEMON_INFO}>{tag}[{pid}]: {line}")
}
/// How long a syslog send may wait for room in journald's queue. dnsmasq runs
/// its lease scripts one at a time, so a send that never returns would stall
/// every later lease event.
const SYSLOG_SEND_TIMEOUT: Duration = Duration::from_secs(1);
/// Send `line` to the syslog socket at `socket` at `info` priority under the
/// identifier `tag`. A full queue fails the send after
/// [`SYSLOG_SEND_TIMEOUT`] instead of waiting for room.
fn send_info(socket: &Path, tag: &str, line: &str) -> std::io::Result<()> {
let datagram = syslog_datagram(tag, std::process::id(), line);
let sock = UnixDatagram::unbound()?;
sock.set_write_timeout(Some(SYSLOG_SEND_TIMEOUT))?;
sock.send_to(datagram.as_bytes(), socket)?;
Ok(())
}
/// The nft input to run for `event`: [`lease_script`], except that a stale /// The nft input to run for `event`: [`lease_script`], except that a stale
/// `del` ([`del_is_stale`]) runs nothing. /// `del` ([`del_is_stale`]) runs nothing.
pub(crate) fn script_to_run(event: &LeaseEvent<'_>) -> Option<String> { pub(crate) fn script_to_run(event: &LeaseEvent<'_>) -> Option<String> {
@ -952,7 +982,10 @@ fn load(args: &[String]) -> Result<()> {
/// that changes no lease, and a `del` for a lease that ran out by the wall /// that changes no lease, and a `del` for a lease that ran out by the wall
/// clock ([`del_is_stale`]) leaves its pair to the pair's own timeout. Every /// clock ([`del_is_stale`]) leaves its pair to the pair's own timeout. Every
/// event that is acted on or skipped leaves one line ([`lease_log_line`]) on /// event that is acted on or skipped leaves one line ([`lease_log_line`]) on
/// stderr, which dnsmasq passes on to its log. /// stderr, which dnsmasq passes on to its log at debug priority, and on the
/// syslog socket at info priority under the identifier `egress-lease`. The log
/// shipper drops debug lines, so the second copy is the one a hive's operator
/// can search for.
fn lease(args: &[String]) -> Result<()> { fn lease(args: &[String]) -> Result<()> {
let [action, mac, ip, ..] = args else { let [action, mac, ip, ..] = args else {
return Ok(()); return Ok(());
@ -978,7 +1011,11 @@ fn lease(args: &[String]) -> Result<()> {
remaining_secs: remaining, remaining_secs: remaining,
now, now,
}; };
eprintln!("{}", lease_log_line(&event, port.as_deref())); let line = lease_log_line(&event, port.as_deref());
eprintln!("{line}");
if let Err(e) = send_info(Path::new(SYSLOG_SOCKET), LEASE, &line) {
eprintln!("{LEASE}: the journal did not take the line: {e}");
}
match script_to_run(&event) { match script_to_run(&event) {
Some(script) => nft(&script), Some(script) => nft(&script),
None => Ok(()), None => Ok(()),
@ -1528,6 +1565,66 @@ mod tests {
assert!(fresh.ends_with("expires=9000 now=8200 applied"), "{fresh}"); assert!(fresh.ends_with("expires=9000 now=8200 applied"), "{fresh}");
} }
#[test]
fn a_lease_line_goes_to_syslog_at_info_under_its_own_identifier() {
assert_eq!(
syslog_datagram(LEASE, 42, "egress-lease: add mac=02:00:00:00:00:0a"),
"<30>egress-lease[42]: egress-lease: add mac=02:00:00:00:00:0a"
);
}
#[test]
fn the_datagram_reaches_the_syslog_socket() {
let dir = std::env::temp_dir().join(format!("egress-lease-syslog-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let socket = dir.join("log");
let server = UnixDatagram::bind(&socket).unwrap();
send_info(&socket, LEASE, "egress-lease: add mac=m").unwrap();
let mut buf = [0_u8; 256];
let len = server.recv(&mut buf).unwrap();
let got = std::str::from_utf8(&buf[..len]).unwrap();
assert!(got.starts_with("<30>egress-lease["), "{got}");
assert!(got.ends_with("]: egress-lease: add mac=m"), "{got}");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn a_full_syslog_queue_fails_the_send_instead_of_hanging() {
let dir = std::env::temp_dir().join(format!("egress-lease-full-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let socket = dir.join("log");
// A bound socket nobody reads: fill its queue until a send would block.
// One datagram per fresh sender, so the receiver's queue length is the
// limit that fills, not one sender's own buffer.
let _server = UnixDatagram::bind(&socket).unwrap();
let mut sent = 0_u32;
loop {
let filler = UnixDatagram::unbound().unwrap();
filler.set_nonblocking(true).unwrap();
if filler.send_to(&[0_u8; 16], &socket).is_err() {
break;
}
sent += 1;
assert!(sent < 100_000, "the queue never filled");
}
let path = socket.clone();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
tx.send(send_info(&path, LEASE, "egress-lease: add mac=m"))
.ok();
});
let result = rx
.recv_timeout(Duration::from_secs(10))
.expect("send_info hung on a full queue");
assert!(result.is_err());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn a_missing_syslog_socket_is_an_error_the_caller_can_report() {
assert!(send_info(Path::new("/nonexistent/log"), LEASE, "x").is_err());
}
#[test] #[test]
fn seeding_skips_expired_and_malformed_leases() { fn seeding_skips_expired_and_malformed_leases() {
let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\ let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\

View file

@ -288,8 +288,14 @@ pkgs.testers.runNixOSTest {
def acks_of(mac_addr): def acks_of(mac_addr):
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip()) return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
# What `hive-priv egress-lease` leaves in the journal: one line per lease event, under its
# own identifier at info priority, which is as low as the log shipper reads. `-u dnsmasq`
# would also match dnsmasq's debug-priority copy of the same stderr line, which the shipper
# drops and which would count each event twice.
LEASE_LOG = "journalctl -t egress-lease -p info -o cat --no-pager"
def stale_dels(): def stale_dels():
return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip() return host.succeed(LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
def applied_dels(who=""): def applied_dels(who=""):
"""How many deletes the lease script applied so far, all or `mac=<mac> ` only. """How many deletes the lease script applied so far, all or `mac=<mac> ` only.
@ -297,7 +303,7 @@ pkgs.testers.runNixOSTest {
A count is taken before the step under test and compared after: the A count is taken before the step under test and compared after: the
journal holds the lines of every earlier subtest. journal holds the lines of every earlier subtest.
""" """
return int(host.succeed(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del {who}.* applied' || true").strip()) return int(host.succeed(f"{LEASE_LOG} | grep -c 'egress-lease: del {who}.* applied' || true").strip())
host.start() host.start()
host.wait_for_unit("multi-user.target") host.wait_for_unit("multi-user.target")
@ -336,6 +342,10 @@ pkgs.testers.runNixOSTest {
port = found.group(1) port = found.group(1)
print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}") print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}")
assert not spoof_lines(), spoof_lines() assert not spoof_lines(), spoof_lines()
# Each lease left its line in the journal where the shipper can read it, with the
# container's MAC and address in it.
for mac_addr, ip_addr in ((MAC_A, IP_A), (MAC_B, IP_B)):
host.wait_until_succeeds(f"{LEASE_LOG} | grep -E 'egress-lease: (add|old) mac={mac_addr} ip={ip_addr} '")
with subtest("a container sending from its own address reaches the host and the other container"): with subtest("a container sending from its own address reaches the host and the other container"):
assert ct_ok("a", "ping -c1 -W2 ${gateway}") assert ct_ok("a", "ping -c1 -W2 ${gateway}")
@ -509,7 +519,7 @@ pkgs.testers.runNixOSTest {
host.succeed("date -s @$(( $(date +%s) + 7200 ))") host.succeed("date -s @$(( $(date +%s) + 7200 ))")
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr. # Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
ct("a", "perl ${bootp} " + MAC_A) ct("a", "perl ${bootp} " + MAC_A)
host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60) host.wait_until_succeeds("test $(" + LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease" assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
@ -537,7 +547,7 @@ pkgs.testers.runNixOSTest {
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30) host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
host.wait_until_succeeds( host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released), "test $(" + LEASE_LOG + " | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
timeout=30, timeout=30,
) )
# a asks again, as its container would after a restart. # a asks again, as its container would after a restart.