From d9551415bfb1b41a15a1714b39c40562a5530612 Mon Sep 17 00:00:00 2001 From: maia Date: Thu, 19 Nov 2026 15:28:57 +0100 Subject: [PATCH] hive-egress: the lease hook's lines reach the journal at info under egress-lease dnsmasq relays the hook's stderr at LOG_DEBUG as dnsmasq-script, and the field log shipper's journald receiver keeps info and above, so no lease add or renewal was visible in the field. The hook now also writes each line to /dev/log at info priority with identifier egress-lease. The stderr print stays. The VM test reads the lines back with journalctl -t egress-lease -p info. --- docs/4-implementation/networking/network.md | 12 ++- hive-priv/src/egress.rs | 101 +++++++++++++++++++- nix/vm-tests/hive-egress-antispoof.nix | 18 +++- 3 files changed, 122 insertions(+), 9 deletions(-) diff --git a/docs/4-implementation/networking/network.md b/docs/4-implementation/networking/network.md index 2ff58f4fb..a23bf938d 100644 --- a/docs/4-implementation/networking/network.md +++ b/docs/4-implementation/networking/network.md @@ -857,9 +857,15 @@ source check lives in that table and the host-side filtering stays in `DNSMASQ_LEASE_EXPIRES` lies in the past does nothing, and the pair leaves the table at its own timeout, as the container's lease ends. A `del` for a lease that hasn't expired (a release) deletes the pair at once, as does one without - a readable expiry. `egress-lease` logs each `add`, `old` and `del` to dnsmasq's - log with the prefix `egress-lease:`, the MAC, address, port, expiry, and for - `del` whether it was `applied` or `skipped (stale)`. + a readable expiry. `egress-lease` logs each `add`, `old` and `del` to the + journal under the identifier `egress-lease`, at info priority, with the prefix + `egress-lease:`, the MAC, address, port, expiry, and for `del` whether it was + `applied` or `skipped (stale)`. It also prints the line on stderr, which + dnsmasq logs at debug priority as `dnsmasq-script`. The log shipper reads info + and above, so search the store for the `egress-lease` identifier, not for the + dnsmasq copy. The journal send waits at most a second for room in journald's + queue, because dnsmasq runs its lease scripts one at a time. A send that + fails or times out costs that one journal line and nothing else. - **The window.** The binding takes the MAC the container's link has when the command first looks, normally within a second of the veth appearing. Until then root in the container could change its own MAC, but in that time only diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 1b94cec9d..61e2cc988 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -32,6 +32,7 @@ use std::fmt::Write as _; use std::fs::{File, OpenOptions, TryLockError}; use std::io::{Read, Write as _}; +use std::os::unix::net::UnixDatagram; use std::path::Path; use std::process::{Command, Output, Stdio}; use std::sync::mpsc; @@ -501,6 +502,35 @@ pub(crate) fn lease_log_line(event: &LeaseEvent<'_>, port: Option<&str>) -> Stri line } +/// The socket journald listens on for syslog datagrams. +const SYSLOG_SOCKET: &str = "/dev/log"; + +/// The syslog priority of an `info` line from the `daemon` facility: facility +/// 3 × 8 plus severity 6. +const SYSLOG_DAEMON_INFO: u8 = 30; + +/// One syslog datagram, `tag[pid]: line`. journald files it under +/// `SYSLOG_IDENTIFIER=tag` at the datagram's priority. +pub(crate) fn syslog_datagram(tag: &str, pid: u32, line: &str) -> String { + format!("<{SYSLOG_DAEMON_INFO}>{tag}[{pid}]: {line}") +} + +/// How long a syslog send may wait for room in journald's queue. dnsmasq runs +/// its lease scripts one at a time, so a send that never returns would stall +/// every later lease event. +const SYSLOG_SEND_TIMEOUT: Duration = Duration::from_secs(1); + +/// Send `line` to the syslog socket at `socket` at `info` priority under the +/// identifier `tag`. A full queue fails the send after +/// [`SYSLOG_SEND_TIMEOUT`] instead of waiting for room. +fn send_info(socket: &Path, tag: &str, line: &str) -> std::io::Result<()> { + let datagram = syslog_datagram(tag, std::process::id(), line); + let sock = UnixDatagram::unbound()?; + sock.set_write_timeout(Some(SYSLOG_SEND_TIMEOUT))?; + sock.send_to(datagram.as_bytes(), socket)?; + Ok(()) +} + /// The nft input to run for `event`: [`lease_script`], except that a stale /// `del` ([`del_is_stale`]) runs nothing. pub(crate) fn script_to_run(event: &LeaseEvent<'_>) -> Option { @@ -952,7 +982,10 @@ fn load(args: &[String]) -> Result<()> { /// that changes no lease, and a `del` for a lease that ran out by the wall /// clock ([`del_is_stale`]) leaves its pair to the pair's own timeout. Every /// event that is acted on or skipped leaves one line ([`lease_log_line`]) on -/// stderr, which dnsmasq passes on to its log. +/// stderr, which dnsmasq passes on to its log at debug priority, and on the +/// syslog socket at info priority under the identifier `egress-lease`. The log +/// shipper drops debug lines, so the second copy is the one a hive's operator +/// can search for. fn lease(args: &[String]) -> Result<()> { let [action, mac, ip, ..] = args else { return Ok(()); @@ -978,7 +1011,11 @@ fn lease(args: &[String]) -> Result<()> { remaining_secs: remaining, now, }; - eprintln!("{}", lease_log_line(&event, port.as_deref())); + let line = lease_log_line(&event, port.as_deref()); + eprintln!("{line}"); + if let Err(e) = send_info(Path::new(SYSLOG_SOCKET), LEASE, &line) { + eprintln!("{LEASE}: the journal did not take the line: {e}"); + } match script_to_run(&event) { Some(script) => nft(&script), None => Ok(()), @@ -1528,6 +1565,66 @@ mod tests { assert!(fresh.ends_with("expires=9000 now=8200 applied"), "{fresh}"); } + #[test] + fn a_lease_line_goes_to_syslog_at_info_under_its_own_identifier() { + assert_eq!( + syslog_datagram(LEASE, 42, "egress-lease: add mac=02:00:00:00:00:0a"), + "<30>egress-lease[42]: egress-lease: add mac=02:00:00:00:00:0a" + ); + } + + #[test] + fn the_datagram_reaches_the_syslog_socket() { + let dir = std::env::temp_dir().join(format!("egress-lease-syslog-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("log"); + let server = UnixDatagram::bind(&socket).unwrap(); + send_info(&socket, LEASE, "egress-lease: add mac=m").unwrap(); + let mut buf = [0_u8; 256]; + let len = server.recv(&mut buf).unwrap(); + let got = std::str::from_utf8(&buf[..len]).unwrap(); + assert!(got.starts_with("<30>egress-lease["), "{got}"); + assert!(got.ends_with("]: egress-lease: add mac=m"), "{got}"); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_full_syslog_queue_fails_the_send_instead_of_hanging() { + let dir = std::env::temp_dir().join(format!("egress-lease-full-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("log"); + // A bound socket nobody reads: fill its queue until a send would block. + // One datagram per fresh sender, so the receiver's queue length is the + // limit that fills, not one sender's own buffer. + let _server = UnixDatagram::bind(&socket).unwrap(); + let mut sent = 0_u32; + loop { + let filler = UnixDatagram::unbound().unwrap(); + filler.set_nonblocking(true).unwrap(); + if filler.send_to(&[0_u8; 16], &socket).is_err() { + break; + } + sent += 1; + assert!(sent < 100_000, "the queue never filled"); + } + let path = socket.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + tx.send(send_info(&path, LEASE, "egress-lease: add mac=m")) + .ok(); + }); + let result = rx + .recv_timeout(Duration::from_secs(10)) + .expect("send_info hung on a full queue"); + assert!(result.is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_missing_syslog_socket_is_an_error_the_caller_can_report() { + assert!(send_info(Path::new("/nonexistent/log"), LEASE, "x").is_err()); + } + #[test] fn seeding_skips_expired_and_malformed_leases() { let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\ diff --git a/nix/vm-tests/hive-egress-antispoof.nix b/nix/vm-tests/hive-egress-antispoof.nix index a8510a9d6..fc5d955bf 100644 --- a/nix/vm-tests/hive-egress-antispoof.nix +++ b/nix/vm-tests/hive-egress-antispoof.nix @@ -288,8 +288,14 @@ pkgs.testers.runNixOSTest { def acks_of(mac_addr): return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip()) + # What `hive-priv egress-lease` leaves in the journal: one line per lease event, under its + # own identifier at info priority, which is as low as the log shipper reads. `-u dnsmasq` + # would also match dnsmasq's debug-priority copy of the same stderr line, which the shipper + # drops and which would count each event twice. + LEASE_LOG = "journalctl -t egress-lease -p info -o cat --no-pager" + def stale_dels(): - return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip() + return host.succeed(LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)' || true").strip() def applied_dels(who=""): """How many deletes the lease script applied so far, all or `mac= ` only. @@ -297,7 +303,7 @@ pkgs.testers.runNixOSTest { A count is taken before the step under test and compared after: the journal holds the lines of every earlier subtest. """ - return int(host.succeed(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del {who}.* applied' || true").strip()) + return int(host.succeed(f"{LEASE_LOG} | grep -c 'egress-lease: del {who}.* applied' || true").strip()) host.start() host.wait_for_unit("multi-user.target") @@ -336,6 +342,10 @@ pkgs.testers.runNixOSTest { port = found.group(1) print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}") assert not spoof_lines(), spoof_lines() + # Each lease left its line in the journal where the shipper can read it, with the + # container's MAC and address in it. + for mac_addr, ip_addr in ((MAC_A, IP_A), (MAC_B, IP_B)): + host.wait_until_succeeds(f"{LEASE_LOG} | grep -E 'egress-lease: (add|old) mac={mac_addr} ip={ip_addr} '") with subtest("a container sending from its own address reaches the host and the other container"): assert ct_ok("a", "ping -c1 -W2 ${gateway}") @@ -509,7 +519,7 @@ pkgs.testers.runNixOSTest { host.succeed("date -s @$(( $(date +%s) + 7200 ))") # Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr. ct("a", "perl ${bootp} " + MAC_A) - host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60) + host.wait_until_succeeds("test $(" + LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60) host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'") host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease" @@ -537,7 +547,7 @@ pkgs.testers.runNixOSTest { host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30) host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'") host.wait_until_succeeds( - "test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released), + "test $(" + LEASE_LOG + " | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released), timeout=30, ) # a asks again, as its container would after a restart.