hive-egress: the lease hook's lines reach the journal at info under egress-lease
dnsmasq relays the hook's stderr at LOG_DEBUG as dnsmasq-script, and the field log shipper's journald receiver keeps info and above, so no lease add or renewal was visible in the field. The hook now also writes each line to /dev/log at info priority with identifier egress-lease. The stderr print stays. The VM test reads the lines back with journalctl -t egress-lease -p info.
This commit is contained in:
parent
7efa8c8ea7
commit
d9551415bf
3 changed files with 122 additions and 9 deletions
|
|
@ -857,9 +857,15 @@ source check lives in that table and the host-side filtering stays in
|
|||
`DNSMASQ_LEASE_EXPIRES` lies in the past does nothing, and the pair leaves the
|
||||
table at its own timeout, as the container's lease ends. A `del` for a lease
|
||||
that hasn't expired (a release) deletes the pair at once, as does one without
|
||||
a readable expiry. `egress-lease` logs each `add`, `old` and `del` to dnsmasq's
|
||||
log with the prefix `egress-lease:`, the MAC, address, port, expiry, and for
|
||||
`del` whether it was `applied` or `skipped (stale)`.
|
||||
a readable expiry. `egress-lease` logs each `add`, `old` and `del` to the
|
||||
journal under the identifier `egress-lease`, at info priority, with the prefix
|
||||
`egress-lease:`, the MAC, address, port, expiry, and for `del` whether it was
|
||||
`applied` or `skipped (stale)`. It also prints the line on stderr, which
|
||||
dnsmasq logs at debug priority as `dnsmasq-script`. The log shipper reads info
|
||||
and above, so search the store for the `egress-lease` identifier, not for the
|
||||
dnsmasq copy. The journal send waits at most a second for room in journald's
|
||||
queue, because dnsmasq runs its lease scripts one at a time. A send that
|
||||
fails or times out costs that one journal line and nothing else.
|
||||
- **The window.** The binding takes the MAC the container's link has when the
|
||||
command first looks, normally within a second of the veth appearing. Until
|
||||
then root in the container could change its own MAC, but in that time only
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
use std::fmt::Write as _;
|
||||
use std::fs::{File, OpenOptions, TryLockError};
|
||||
use std::io::{Read, Write as _};
|
||||
use std::os::unix::net::UnixDatagram;
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Output, Stdio};
|
||||
use std::sync::mpsc;
|
||||
|
|
@ -501,6 +502,35 @@ pub(crate) fn lease_log_line(event: &LeaseEvent<'_>, port: Option<&str>) -> Stri
|
|||
line
|
||||
}
|
||||
|
||||
/// The socket journald listens on for syslog datagrams.
|
||||
const SYSLOG_SOCKET: &str = "/dev/log";
|
||||
|
||||
/// The syslog priority of an `info` line from the `daemon` facility: facility
|
||||
/// 3 × 8 plus severity 6.
|
||||
const SYSLOG_DAEMON_INFO: u8 = 30;
|
||||
|
||||
/// One syslog datagram, `<priority>tag[pid]: line`. journald files it under
|
||||
/// `SYSLOG_IDENTIFIER=tag` at the datagram's priority.
|
||||
pub(crate) fn syslog_datagram(tag: &str, pid: u32, line: &str) -> String {
|
||||
format!("<{SYSLOG_DAEMON_INFO}>{tag}[{pid}]: {line}")
|
||||
}
|
||||
|
||||
/// How long a syslog send may wait for room in journald's queue. dnsmasq runs
|
||||
/// its lease scripts one at a time, so a send that never returns would stall
|
||||
/// every later lease event.
|
||||
const SYSLOG_SEND_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
|
||||
/// Send `line` to the syslog socket at `socket` at `info` priority under the
|
||||
/// identifier `tag`. A full queue fails the send after
|
||||
/// [`SYSLOG_SEND_TIMEOUT`] instead of waiting for room.
|
||||
fn send_info(socket: &Path, tag: &str, line: &str) -> std::io::Result<()> {
|
||||
let datagram = syslog_datagram(tag, std::process::id(), line);
|
||||
let sock = UnixDatagram::unbound()?;
|
||||
sock.set_write_timeout(Some(SYSLOG_SEND_TIMEOUT))?;
|
||||
sock.send_to(datagram.as_bytes(), socket)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The nft input to run for `event`: [`lease_script`], except that a stale
|
||||
/// `del` ([`del_is_stale`]) runs nothing.
|
||||
pub(crate) fn script_to_run(event: &LeaseEvent<'_>) -> Option<String> {
|
||||
|
|
@ -952,7 +982,10 @@ fn load(args: &[String]) -> Result<()> {
|
|||
/// that changes no lease, and a `del` for a lease that ran out by the wall
|
||||
/// clock ([`del_is_stale`]) leaves its pair to the pair's own timeout. Every
|
||||
/// event that is acted on or skipped leaves one line ([`lease_log_line`]) on
|
||||
/// stderr, which dnsmasq passes on to its log.
|
||||
/// stderr, which dnsmasq passes on to its log at debug priority, and on the
|
||||
/// syslog socket at info priority under the identifier `egress-lease`. The log
|
||||
/// shipper drops debug lines, so the second copy is the one a hive's operator
|
||||
/// can search for.
|
||||
fn lease(args: &[String]) -> Result<()> {
|
||||
let [action, mac, ip, ..] = args else {
|
||||
return Ok(());
|
||||
|
|
@ -978,7 +1011,11 @@ fn lease(args: &[String]) -> Result<()> {
|
|||
remaining_secs: remaining,
|
||||
now,
|
||||
};
|
||||
eprintln!("{}", lease_log_line(&event, port.as_deref()));
|
||||
let line = lease_log_line(&event, port.as_deref());
|
||||
eprintln!("{line}");
|
||||
if let Err(e) = send_info(Path::new(SYSLOG_SOCKET), LEASE, &line) {
|
||||
eprintln!("{LEASE}: the journal did not take the line: {e}");
|
||||
}
|
||||
match script_to_run(&event) {
|
||||
Some(script) => nft(&script),
|
||||
None => Ok(()),
|
||||
|
|
@ -1528,6 +1565,66 @@ mod tests {
|
|||
assert!(fresh.ends_with("expires=9000 now=8200 applied"), "{fresh}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lease_line_goes_to_syslog_at_info_under_its_own_identifier() {
|
||||
assert_eq!(
|
||||
syslog_datagram(LEASE, 42, "egress-lease: add mac=02:00:00:00:00:0a"),
|
||||
"<30>egress-lease[42]: egress-lease: add mac=02:00:00:00:00:0a"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_datagram_reaches_the_syslog_socket() {
|
||||
let dir = std::env::temp_dir().join(format!("egress-lease-syslog-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let socket = dir.join("log");
|
||||
let server = UnixDatagram::bind(&socket).unwrap();
|
||||
send_info(&socket, LEASE, "egress-lease: add mac=m").unwrap();
|
||||
let mut buf = [0_u8; 256];
|
||||
let len = server.recv(&mut buf).unwrap();
|
||||
let got = std::str::from_utf8(&buf[..len]).unwrap();
|
||||
assert!(got.starts_with("<30>egress-lease["), "{got}");
|
||||
assert!(got.ends_with("]: egress-lease: add mac=m"), "{got}");
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_syslog_queue_fails_the_send_instead_of_hanging() {
|
||||
let dir = std::env::temp_dir().join(format!("egress-lease-full-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let socket = dir.join("log");
|
||||
// A bound socket nobody reads: fill its queue until a send would block.
|
||||
// One datagram per fresh sender, so the receiver's queue length is the
|
||||
// limit that fills, not one sender's own buffer.
|
||||
let _server = UnixDatagram::bind(&socket).unwrap();
|
||||
let mut sent = 0_u32;
|
||||
loop {
|
||||
let filler = UnixDatagram::unbound().unwrap();
|
||||
filler.set_nonblocking(true).unwrap();
|
||||
if filler.send_to(&[0_u8; 16], &socket).is_err() {
|
||||
break;
|
||||
}
|
||||
sent += 1;
|
||||
assert!(sent < 100_000, "the queue never filled");
|
||||
}
|
||||
let path = socket.clone();
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
tx.send(send_info(&path, LEASE, "egress-lease: add mac=m"))
|
||||
.ok();
|
||||
});
|
||||
let result = rx
|
||||
.recv_timeout(Duration::from_secs(10))
|
||||
.expect("send_info hung on a full queue");
|
||||
assert!(result.is_err());
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_syslog_socket_is_an_error_the_caller_can_report() {
|
||||
assert!(send_info(Path::new("/nonexistent/log"), LEASE, "x").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seeding_skips_expired_and_malformed_leases() {
|
||||
let leases = "2000 02:00:00:00:00:0a 10.42.0.10 a *\n\
|
||||
|
|
|
|||
|
|
@ -288,8 +288,14 @@ pkgs.testers.runNixOSTest {
|
|||
def acks_of(mac_addr):
|
||||
return int(host.succeed("echo " + acks_of_cmd(mac_addr)).strip())
|
||||
|
||||
# What `hive-priv egress-lease` leaves in the journal: one line per lease event, under its
|
||||
# own identifier at info priority, which is as low as the log shipper reads. `-u dnsmasq`
|
||||
# would also match dnsmasq's debug-priority copy of the same stderr line, which the shipper
|
||||
# drops and which would count each event twice.
|
||||
LEASE_LOG = "journalctl -t egress-lease -p info -o cat --no-pager"
|
||||
|
||||
def stale_dels():
|
||||
return host.succeed("journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
|
||||
return host.succeed(LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)' || true").strip()
|
||||
|
||||
def applied_dels(who=""):
|
||||
"""How many deletes the lease script applied so far, all or `mac=<mac> ` only.
|
||||
|
|
@ -297,7 +303,7 @@ pkgs.testers.runNixOSTest {
|
|||
A count is taken before the step under test and compared after: the
|
||||
journal holds the lines of every earlier subtest.
|
||||
"""
|
||||
return int(host.succeed(f"journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del {who}.* applied' || true").strip())
|
||||
return int(host.succeed(f"{LEASE_LOG} | grep -c 'egress-lease: del {who}.* applied' || true").strip())
|
||||
|
||||
host.start()
|
||||
host.wait_for_unit("multi-user.target")
|
||||
|
|
@ -336,6 +342,10 @@ pkgs.testers.runNixOSTest {
|
|||
port = found.group(1)
|
||||
print(f"tolerated start-up refusal, {(stamp - QUIET_AFTER_US[port]) / 1e6:+.3f}s from the end of {port}'s settle: {message}")
|
||||
assert not spoof_lines(), spoof_lines()
|
||||
# Each lease left its line in the journal where the shipper can read it, with the
|
||||
# container's MAC and address in it.
|
||||
for mac_addr, ip_addr in ((MAC_A, IP_A), (MAC_B, IP_B)):
|
||||
host.wait_until_succeeds(f"{LEASE_LOG} | grep -E 'egress-lease: (add|old) mac={mac_addr} ip={ip_addr} '")
|
||||
|
||||
with subtest("a container sending from its own address reaches the host and the other container"):
|
||||
assert ct_ok("a", "ping -c1 -W2 ${gateway}")
|
||||
|
|
@ -509,7 +519,7 @@ pkgs.testers.runNixOSTest {
|
|||
host.succeed("date -s @$(( $(date +%s) + 7200 ))")
|
||||
# Any DHCP packet makes dnsmasq prune: a DISCOVER with a's own chaddr.
|
||||
ct("a", "perl ${bootp} " + MAC_A)
|
||||
host.wait_until_succeeds("test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
|
||||
host.wait_until_succeeds("test $(" + LEASE_LOG + " | grep -c 'egress-lease: del .* skipped (stale)') -ge " + str(skipped + 2), timeout=60)
|
||||
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'")
|
||||
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
||||
assert ct_ok("a", "ping -c1 -W2 ${gateway}"), "a reaches the host after dnsmasq forgot its lease"
|
||||
|
|
@ -537,7 +547,7 @@ pkgs.testers.runNixOSTest {
|
|||
host.wait_until_fails(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_A} . {IP_A}'", timeout=30)
|
||||
host.succeed(f"nft list set {TABLE} lease_v4 | grep -q '{MAC_B} . {IP_B}'")
|
||||
host.wait_until_succeeds(
|
||||
"test $(journalctl -u dnsmasq -o cat --no-pager | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
|
||||
"test $(" + LEASE_LOG + " | grep -c 'egress-lease: del mac=" + MAC_A + " .* applied') -gt " + str(released),
|
||||
timeout=30,
|
||||
)
|
||||
# a asks again, as its container would after a restart.
|
||||
|
|
|
|||
Loading…
Reference in a new issue