Watch
0
0
Fork
You've already forked hyperhive
0

hive-egress: the lease hook's lines reach the journal at info under egress-lease

dnsmasq relays the hook's stderr at LOG_DEBUG as dnsmasq-script, and the
field log shipper's journald receiver keeps info and above, so no lease
add or renewal was visible in the field. The hook now also writes each
line to /dev/log at info priority with identifier egress-lease. The
stderr print stays. The VM test reads the lines back with
journalctl -t egress-lease -p info.
This commit is contained in:
maia 2026-11-19 15:28:57 +01:00 • committed by mara
commit d9551415bf
3 changed files with 122 additions and 9 deletions

View file

@ -857,9 +857,15 @@ source check lives in that table and the host-side filtering stays in
`DNSMASQ_LEASE_EXPIRES` lies in the past does nothing, and the pair leaves the
table at its own timeout, as the container's lease ends. A `del` for a lease
that hasn't expired (a release) deletes the pair at once, as does one without
a readable expiry. `egress-lease` logs each `add`, `old` and `del` to dnsmasq's
log with the prefix `egress-lease:`, the MAC, address, port, expiry, and for
`del` whether it was `applied` or `skipped (stale)`.
a readable expiry. `egress-lease` logs each `add`, `old` and `del` to the
journal under the identifier `egress-lease`, at info priority, with the prefix
`egress-lease:`, the MAC, address, port, expiry, and for `del` whether it was
`applied` or `skipped (stale)`. It also prints the line on stderr, which
dnsmasq logs at debug priority as `dnsmasq-script`. The log shipper reads info
and above, so search the store for the `egress-lease` identifier, not for the
dnsmasq copy. The journal send waits at most a second for room in journald's
queue, because dnsmasq runs its lease scripts one at a time. A send that
fails or times out costs that one journal line and nothing else.
- **The window.** The binding takes the MAC the container's link has when the
command first looks, normally within a second of the veth appearing. Until
then root in the container could change its own MAC, but in that time only

View file

@ -32,6 +32,7 @@
use std::fmt::Write as _;
use std::fs::{File, OpenOptions, TryLockError};
use std::io::{Read, Write as _};
use std::os::unix::net::UnixDatagram;
use std::path::Path;
use std::process::{Command, Output, Stdio};
use std::sync::mpsc;
@ -501,6 +502,35 @@ pub(crate) fn lease_log_line(event: &LeaseEvent<'_>, port: Option<&str>) -> Stri
line
}
/// The socket journald listens on for syslog datagrams.
const SYSLOG_SOCKET: &str = "/dev/log";
/// The syslog priority of an `info` line from the `daemon` facility: facility
/// 3 × 8 plus severity 6.
const SYSLOG_DAEMON_INFO: u8 = 30;
/// One syslog datagram, `<priority>tag[pid]: line`. journald files it under
/// `SYSLOG_IDENTIFIER=tag` at the datagram's priority.
pub(crate) fn syslog_datagram(tag: &str, pid: u32, line: &str) -> String {
format!("<{SYSLOG_DAEMON_INFO}>{tag}[{pid}]: {line}")
}
/// How long a syslog send may wait for room in journald's queue. dnsmasq runs
/// its lease scripts one at a time, so a send that never returns would stall
/// every later lease event.
const SYSLOG_SEND_TIMEOUT: Duration = Duration::from_secs(1);
/// Send `line` to the syslog socket at `socket` at `info` priority under the
/// identifier `tag`. A full queue fails the send after
/// [`SYSLOG_SEND_TIMEOUT`] instead of waiting for room.
fn send_info(socket: &Path, tag: &str, line: &str) -> std::io::Result<()> {
let datagram = syslog_datagram(tag, std::process::id(), line);
let sock = UnixDatagram::unbound()?;
sock.set_write_timeout(Some(SYSLOG_SEND_TIMEOUT))?;
sock.send_to(datagram.as_bytes(), socket)?;
Ok(())
}
/// The nft input to run for `event`: [`lease_script`], except that a stale
/// `del` ([`del_is_stale`]) runs nothing.
pub(crate) fn script_to_run(event: &LeaseEvent<'_>) -> Option<String> {
@ -952,7 +982,10 @@ fn load(args: &[String]) -> Result<()> {
/// that changes no lease, and a `del` for a lease that ran out by the wall
/// clock ([`del_is_stale`]) leaves its pair to the pair's own timeout. Every
/// event that is acted on or skipped leaves one line ([`lease_log_line`]) on
/// stderr, which dnsmasq passes on to its log.
/// stderr, which dnsmasq passes on to its log at debug priority, and on the
/// syslog socket at info priority under the identifier `egress-lease`. The log
/// shipper drops debug lines, so the second copy is the one a hive's operator
/// can search for.
fn lease(args: &[String]) -> Result<()> {
let [action, mac, ip, ..] = args else {
return Ok(());
@ -978,7 +1011,11 @@ fn lease(args: &[String]) -> Result<()> {
remaining_secs: remaining,
now,
};
eprintln!("{}", lease_log_line(&event, port.as_deref()));
let line = lease_log_line(&event, port.as_deref());
eprintln!("{line}");
if let Err(e) = send_info(Path::new(SYSLOG_SOCKET), LEASE, &line) {
eprintln!("{LEASE}: the journal did not take the line: {e}");
}
match script_to_run(&event) {
Some(script) => nft(&script),
None => Ok(()),
@ -1528,6 +1565,66 @@ mod tests {
assert!(fresh.ends_with("expires=9000 now=8200 applied"), "{fresh}");
}
#[test]
fn a_lease_line_goes_to_syslog_at_info_under_its_own_identifier() {
assert_eq!(
syslog_datagram(LEASE, 42, "egress-lease: add mac=02:00:00:00:00:0a"),
"<30>egress-lease[42]: egress-lease: add mac=02:00:00:00:00:0a"
);
}
#[test]
fn the_datagram_reaches_the_syslog_socket() {
let dir = std::env::temp_dir().join(format!("egress-lease-syslog-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let socket = dir.join("log");
let server = UnixDatagram::bind(&socket).unwrap();
send_info(&socket, LEASE, "egress-lease: add mac=m").unwrap();
let mut buf = [0_u8; 256];
let len = server.recv(&mut buf).unwrap();
let got = std::str::from_utf8(&buf[..len]).unwrap();
assert!(got.starts_with("<30>egress-lease["), "{got}");
assert!(got.ends_with("]: egress-lease: add mac=m"), "{got}");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn a_full_syslog_queue_fails_the_send_instead_of_hanging() {
let dir = std::env::temp_dir().join(format!("egress-lease-full-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let socket = dir.join("log");
// A bound socket nobody reads: fill its queue until a send would block.
// One datagram per fresh sender, so the receiver's queue length is the
// limit that fills, not one sender's own buffer.
let _server = UnixDatagram::bind(&socket).unwrap();
let mut sent = 0_u32;
loop {
let filler = UnixDatagram::unbound().unwrap();
filler.set_nonblocking(true).unwrap();
if filler.send_to(&[0_u8; 16], &socket).is_err() {
break;
}
sent += 1;
assert!(sent < 100_000, "the queue never filled");
}
let path = socket.clone();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
tx.send(send_info(&path, LEASE, "egress-lease: add mac=m"))
.ok();
});
let result = rx
.recv_timeout(Duration::from_secs(10))
.expect("send_info hung on a full queue");
assert!(result.is_err());
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn a_missing_syslog_socket_is_an_error_the_caller_can_report() {
assert!(send_info(Path::new("/nonexistent/log"), LEASE, "x").is_err());
}
#[test]
fn seeding_skips_expired_and_malformed_leases() {
let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\

View file

@ -288,8 +288,14 @@ pkgs.testers.runNixOSTest {
def acks_of(mac_addr):
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
# What `hive-priv egress-lease` leaves in the journal: one line per lease event, under its
# own identifier at info priority, which is as low as the log shipper reads. `-u dnsmasq`
# would also match dnsmasq's debug-priority copy of the same stderr line, which the shipper
# drops and which would count each event twice.
LEASE_LOG = "journalctl -t egress-lease -p info -o cat --no-pager"
def stale_dels():
return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
return host.succeed(LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
def applied_dels(who=""):
"""How many deletes the lease script applied so far, all or `mac=<mac> ` only.
@ -297,7 +303,7 @@ pkgs.testers.runNixOSTest {
A count is taken before the step under test and compared after: the
journal holds the lines of every earlier subtest.
"""
return int(host.succeed(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del {who}.* applied' || true").strip())
return int(host.succeed(f"{LEASE_LOG} | grep -c 'egress-lease: del {who}.* applied' || true").strip())
host.start()
host.wait_for_unit("multi-user.target")
@ -336,6 +342,10 @@ pkgs.testers.runNixOSTest {
port = found.group(1)
print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}")
assert not spoof_lines(), spoof_lines()
# Each lease left its line in the journal where the shipper can read it, with the
# container's MAC and address in it.
for mac_addr, ip_addr in ((MAC_A, IP_A), (MAC_B, IP_B)):
host.wait_until_succeeds(f"{LEASE_LOG} | grep -E 'egress-lease: (add|old) mac={mac_addr} ip={ip_addr} '")
with subtest("a container sending from its own address reaches the host and the other container"):
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
@ -509,7 +519,7 @@ pkgs.testers.runNixOSTest {
host.succeed("date -s @$(( $(date +%s) + 7200 ))")
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
ct("a", "perl ${bootp} " + MAC_A)
host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
host.wait_until_succeeds("test $(" + LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
@ -537,7 +547,7 @@ pkgs.testers.runNixOSTest {
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
host.wait_until_succeeds(
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
"test $(" + LEASE_LOG + " | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
timeout=30,
)
# a asks again, as its container would after a restart.