diff --git a/hive-c0re/src/main.rs b/hive-c0re/src/main.rs index 67e8acfab..679deb80c 100644 --- a/hive-c0re/src/main.rs +++ b/hive-c0re/src/main.rs @@ -321,6 +321,8 @@ async fn cmd_serve( swarm_agent_status::spawn(std::sync::Arc::clone(&coord)); // Squid's users file: the hashes of this hive's agents' proxy credentials. workers::squid_users::spawn(coord.shutdown_rx()); + // The SSH connections the forward chain lets each agent open. + workers::egress_connections::spawn(coord.shutdown_rx()); // Per-agent bash-tasks file cleanup, and the removal of the old event log, // now run agent-side in the harness (`hive_agent::vacuum`): the files are // agent-owned, so host-side deletes hit PermissionDenied / readonly-db diff --git a/hive-c0re/src/workers/egress_connections.rs b/hive-c0re/src/workers/egress_connections.rs new file mode 100644 index 000000000..955c85172 --- /dev/null +++ b/hive-c0re/src/workers/egress_connections.rs @@ -0,0 +1,802 @@ +//! The SSH connections the hive's forward chain lets agents open: for each +//! agent whose `ssh` switch is on, the addresses behind the `host` of each +//! connection in its permissions record. +//! +//! Every [`INTERVAL`] this reads the permissions the swarm granted this +//! hive's agents ([`crate::permissions`]) and renders one ` +//! ` line per agent, IPv4 address and port into the file named by +//! [`ENV_FILE`]. `hive-priv egress-pin` turns each line into one element of +//! the `agent_allow` set of `nix/host-modules/hive-egress.nix`, keyed by the +//! MAC of the agent's port, which only hive-priv can tell. IPv6 answers are +//! left out: the agent bridge has no IPv6 addressing. +//! +//! A host name is resolved by the system resolver, so it means what it means +//! to the host's other services; an IPv4 literal is taken as it is. Every +//! address a name resolves to is allowed, as the agent may use any of them. +//! An answer is kept for [`RESOLVE_EVERY`], and a host a record newly names +//! is resolved at the next pass. The hosts of a pass are asked concurrently, +//! each for at most [`RESOLVE_TIMEOUT`], so a resolver that hangs delays the +//! file by that long and no more. A name that fails to resolve, or times out, +//! keeps the addresses of its last answer and logs one warning until it +//! resolves again: a resolver outage must not read as the operator removing +//! the connection. An address inside this hive's bridge subnet is dropped, +//! with one warning per host: a flow to it never reaches the forward chain. +//! +//! A failed read of the permissions leaves the file as it is. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::future::Future; +use std::net::{IpAddr, Ipv4Addr}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use futures_util::StreamExt as _; + +use super::squid_users::write_if_changed; +use crate::permissions; +use crate::stats::sweep_health::{self, SweepHealth}; + +/// How often the file is re-rendered, which is how soon a new connection +/// or a flipped `ssh` switch takes effect. +pub const INTERVAL: Duration = Duration::from_secs(30); + +/// How long an answer of the resolver is used before the name is asked again. +pub const RESOLVE_EVERY: Duration = Duration::from_mins(10); + +/// How long one lookup may take before it counts as failed. +pub const RESOLVE_TIMEOUT: Duration = Duration::from_secs(5); + +/// How many lookups of one pass run at the same time. +const CONCURRENT_LOOKUPS: usize = 8; + +/// At [`INTERVAL`] this is ~90 seconds of failing passes. +const FAILURES_BEFORE_WARNING: u32 = 3; + +/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re` +/// from `services.hyperhive.deploy.hive-controller.egress.connectionsFile`; +/// unset means this host's forward chain does not read one. +pub const ENV_FILE: &str = "HIVE_EGRESS_CONNECTIONS_FILE"; + +/// The environment variable that carries the bridge address and prefix +/// (`10.42.0.1/24`); `nix/host-modules/hive-network.nix` sets it for the daemon. +const ENV_SUBNET: &str = "HIVE_NETWORK_SUBNET"; + +/// An IPv4 subnet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Subnet { + network: u32, + mask: u32, +} + +impl Subnet { + /// `/`, where the address may be any host of the subnet. + fn parse(text: &str) -> Option { + let (ip, prefix) = text.split_once('/')?; + let prefix: u32 = prefix.parse().ok().filter(|p| *p <= 32)?; + let mask = u32::MAX.checked_shl(32 - prefix).unwrap_or(0); + Some(Self { + network: u32::from(ip.parse::().ok()?) & mask, + mask, + }) + } + + fn contains(self, ip: Ipv4Addr) -> bool { + u32::from(ip) & self.mask == self.network + } +} + +/// One declared connection of an agent: where it goes. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Target { + agent: String, + host: String, + port: u16, +} + +/// The connections of every agent whose `ssh` switch is on. +fn targets(perms: &permissions::Stored) -> BTreeSet { + let mut out = BTreeSet::new(); + for (agent, record) in perms { + if record.ssh != Some(true) || hive_types::Ident::parse(agent).is_err() { + continue; + } + for c in &record.ssh_connections { + out.insert(Target { + agent: agent.clone(), + host: c.host.clone(), + port: c.port, + }); + } + } + out +} + +/// What a host resolved to at its last successful ask. +struct Answer { + /// When it was asked; `None` once an ask failed, so the next pass asks + /// again. + at: Option, + addrs: Vec, +} + +/// What each host resolved to. +#[derive(Default)] +struct Answers { + by_host: HashMap, + /// Hosts whose failure was logged and that have not resolved since. + warned: HashSet, + /// Hosts whose answer holds an address of the bridge subnet, logged once. + warned_bridge: HashSet, +} + +impl Answers { + /// Ask for every host in `hosts` that has no answer yet or whose answer is + /// older than [`RESOLVE_EVERY`], and forget the others. The asks run + /// [`CONCURRENT_LOOKUPS`] at a time, each for at most [`RESOLVE_TIMEOUT`]. + async fn refresh( + &mut self, + hosts: &BTreeSet<&str>, + now: Instant, + bridge: Option, + resolve: F, + ) where + F: Fn(String) -> Fut, + Fut: Future>>, + { + self.by_host.retain(|h, _| hosts.contains(h.as_str())); + self.warned.retain(|h| hosts.contains(h.as_str())); + self.warned_bridge.retain(|h| hosts.contains(h.as_str())); + let due: Vec = hosts + .iter() + .filter(|host| { + self.by_host + .get(**host) + .and_then(|a| a.at) + .is_none_or(|at| now.duration_since(at) >= RESOLVE_EVERY) + }) + .map(|host| (*host).to_owned()) + .collect(); + let resolve = &resolve; + let asked: Vec<(String, Result>)> = futures_util::stream::iter(due) + .map(|host| async move { + let answer = ask(&host, resolve).await; + (host, answer) + }) + .buffer_unordered(CONCURRENT_LOOKUPS) + .collect() + .await; + for (host, result) in asked { + let host = host.as_str(); + match result { + Ok(addrs) => { + self.warned.remove(host); + let (usable, inside) = usable_v4(&addrs, bridge); + if inside.is_empty() { + self.warned_bridge.remove(host); + } else if self.warned_bridge.insert(host.to_owned()) { + tracing::warn!( + host, + addresses = ?inside, + "egress connections: the host resolves into the hive's bridge \ + subnet; those addresses are not allowed" + ); + } + self.by_host.insert( + host.to_owned(), + Answer { + at: Some(now), + addrs: usable, + }, + ); + } + Err(e) => { + if self.warned.insert(host.to_owned()) { + tracing::warn!( + host, + error = ?e, + "egress connections: resolving failed, keeping the last answer" + ); + } + if let Some(answer) = self.by_host.get_mut(host) { + answer.at = None; + } + } + } + } + } + + fn addrs(&self, host: &str) -> &[Ipv4Addr] { + self.by_host.get(host).map_or(&[], |a| a.addrs.as_slice()) + } +} + +/// `host` as an address when it is a literal, else what `resolve` answers +/// within [`RESOLVE_TIMEOUT`]. +async fn ask(host: &str, resolve: &F) -> Result> +where + F: Fn(String) -> Fut, + Fut: Future>>, +{ + if let Ok(ip) = host.parse::() { + return Ok(vec![ip]); + } + tokio::time::timeout(RESOLVE_TIMEOUT, resolve(host.to_owned())) + .await + .map_err(|_| anyhow::anyhow!("resolving {host} took more than {RESOLVE_TIMEOUT:?}"))? +} + +/// The IPv4 addresses among `addrs` that a remote machine can have, and the +/// ones dropped for lying inside the `bridge` subnet. +fn usable_v4(addrs: &[IpAddr], bridge: Option) -> (Vec, Vec) { + let mut v4: Vec = addrs + .iter() + .filter_map(|a| match a { + IpAddr::V4(v4) => Some(*v4), + IpAddr::V6(_) => None, + }) + .filter(|a| { + !(a.is_loopback() + || a.is_link_local() + || a.is_unspecified() + || a.is_multicast() + || a.is_broadcast()) + }) + .collect(); + v4.sort_unstable(); + v4.dedup(); + v4.into_iter() + .partition(|a| !bridge.is_some_and(|b| b.contains(*a))) +} + +/// The file's text for `targets` under the answers in `answers`, sorted. +fn render(targets: &BTreeSet, answers: &Answers) -> String { + let mut lines = BTreeSet::new(); + for t in targets { + for ip in answers.addrs(&t.host) { + lines.insert(format!("{} {ip} {}\n", t.agent, t.port)); + } + } + lines.into_iter().collect() +} + +/// One pass: refresh the answers for the hosts `perms` names and write the file +/// if it changed. +async fn pass_with( + perms: &permissions::Stored, + answers: &mut Answers, + file: &Path, + now: Instant, + bridge: Option, + resolve: F, +) -> Result<()> +where + F: Fn(String) -> Fut, + Fut: Future>>, +{ + let wanted = targets(perms); + let hosts: BTreeSet<&str> = wanted.iter().map(|t| t.host.as_str()).collect(); + answers.refresh(&hosts, now, bridge, resolve).await; + let text = render(&wanted, answers); + if write_if_changed(file, &text)? { + tracing::info!( + file = %file.display(), + "egress connections: wrote the list, {} allowed addresses", + text.lines().count() + ); + } + Ok(()) +} + +/// The system resolver's addresses for `host`. +async fn system_resolve(host: String) -> Result> { + let found = tokio::net::lookup_host((host.as_str(), 0)) + .await + .with_context(|| format!("resolving {host}"))?; + Ok(found.map(|a| a.ip()).collect()) +} + +fn env_path(name: &str) -> Option { + std::env::var_os(name) + .filter(|v| !v.is_empty()) + .map(PathBuf::from) +} + +/// Start the pass loop, when this host names the file. +pub fn spawn(mut shutdown: tokio::sync::watch::Receiver) { + let Some(file) = env_path(ENV_FILE) else { + return; + }; + let bridge = std::env::var(ENV_SUBNET) + .ok() + .and_then(|s| Subnet::parse(&s)); + if bridge.is_none() { + tracing::warn!( + "egress connections: {ENV_SUBNET} is unset or invalid, so the bridge subnet is not excluded" + ); + } + tokio::spawn(async move { + let mut health = + SweepHealth::new("egress_connections_render", "warn", FAILURES_BEFORE_WARNING); + let mut answers = Answers::default(); + loop { + let result = match permissions::read_at(&permissions::path()) { + Ok(perms) => { + pass_with( + &perms, + &mut answers, + &file, + Instant::now(), + bridge, + system_resolve, + ) + .await + } + Err(e) => { + Err(anyhow::Error::new(e).context("reading the stored swarm permissions")) + } + }; + match result { + Ok(()) => health.record_ok(), + Err(e) => { + tracing::warn!(error = ?e, "egress connections: the pass failed"); + let err = format!("{e:#}"); + health.record_err(|ctx| { + let age = ctx.since_last_ok.map_or_else( + || "no success this session".to_owned(), + |d| format!("last ok {} ago", sweep_health::fmt_age(d)), + ); + format!( + "rendering the agents' allowed ssh connections is failing ({} \ + consecutive, {age}) — the forward chain keeps the list last \ + written: {err}", + ctx.consecutive + ) + }); + } + } + tokio::select! { + () = tokio::time::sleep(INTERVAL) => {} + _ = shutdown.changed() => return, + } + } + }); +} + +#[cfg(test)] +mod tests { + use std::cell::RefCell; + use std::rc::Rc; + + use super::*; + use swarm_wire::agent_permissions::AgentPermissions; + + fn record(json: &str) -> AgentPermissions { + serde_json::from_str(json).expect("decodes") + } + + fn pc2(agent_ssh: &str) -> AgentPermissions { + record(&format!( + r#"{{"ssh":{agent_ssh},"ssh_connections":[{{"name":"k8s-lab-vm","user":"root","host":"muede-pc2","port":2222}}]}}"# + )) + } + + fn ip(s: &str) -> IpAddr { + s.parse().expect("an address") + } + + type ResolveFuture = std::pin::Pin>>>>; + + /// A resolver that answers from `table`, counts its calls and fails for + /// names not in it. + #[derive(Clone, Default)] + struct Dns { + table: Rc>>>, + calls: Rc>>, + /// Hosts whose lookup never answers. + hung: Rc>>, + } + + impl Dns { + fn set(&self, host: &str, addrs: &[&str]) { + self.table + .borrow_mut() + .insert(host.to_owned(), addrs.iter().map(|a| ip(a)).collect()); + } + fn fail(&self, host: &str) { + self.table.borrow_mut().remove(host); + } + fn hang(&self, host: &str) { + self.hung.borrow_mut().insert(host.to_owned()); + } + fn resolver(&self) -> impl Fn(String) -> ResolveFuture + use<> { + let (table, calls, hung) = (self.table.clone(), self.calls.clone(), self.hung.clone()); + move |host| { + calls.borrow_mut().push(host.clone()); + if hung.borrow().contains(&host) { + return Box::pin(std::future::pending()); + } + let answer = table + .borrow() + .get(&host) + .cloned() + .ok_or_else(|| anyhow::anyhow!("no such host")); + Box::pin(std::future::ready(answer)) + } + } + fn calls(&self) -> usize { + self.calls.borrow().len() + } + } + + struct Rig { + dir: tempfile::TempDir, + answers: Answers, + dns: Dns, + start: Instant, + bridge: Option, + } + + impl Rig { + fn new() -> Self { + let rig = Self { + dir: tempfile::tempdir().expect("tempdir"), + answers: Answers::default(), + dns: Dns::default(), + start: Instant::now(), + bridge: None, + }; + // The empty file `systemd-tmpfiles` creates before c0re starts. + std::fs::write(rig.file(), "").expect("write"); + rig + } + fn file(&self) -> PathBuf { + self.dir.path().join("connections") + } + async fn pass(&mut self, perms: &permissions::Stored, after: Duration) -> Result<()> { + let (file, now) = (self.file(), self.start + after); + pass_with( + perms, + &mut self.answers, + &file, + now, + self.bridge, + self.dns.resolver(), + ) + .await + } + fn text(&self) -> String { + std::fs::read_to_string(self.file()).expect("the file is there") + } + } + + const T0: Duration = Duration::ZERO; + + #[tokio::test] + async fn a_name_is_allowed_at_every_address_it_resolves_to() { + let mut rig = Rig::new(); + rig.dns + .set("muede-pc2", &["192.168.23.232", "192.168.23.183"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!( + rig.text(), + "atlas 192.168.23.183 2222\natlas 192.168.23.232 2222\n" + ); + } + + #[tokio::test] + async fn an_ip_literal_passes_through_without_a_lookup() { + let mut rig = Rig::new(); + let perms = permissions::Stored::from([( + "atlas".to_owned(), + record( + r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"10.1.2.3","port":22}]}"#, + ), + )]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 10.1.2.3 22\n"); + assert_eq!(rig.dns.calls(), 0); + } + + #[tokio::test] + async fn the_ssh_switch_off_or_unset_allows_nothing() { + for ssh in ["false", "null"] { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2(ssh))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "", "ssh = {ssh}"); + assert_eq!(rig.dns.calls(), 0, "a switched-off agent costs no lookup"); + } + // Control: the same record with the switch on is allowed. + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + } + + #[tokio::test] + async fn switching_ssh_off_removes_the_agents_lines_and_only_its_own() { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let mut perms = permissions::Stored::from([ + ("atlas".to_owned(), pc2("true")), + ("argus".to_owned(), pc2("true")), + ]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text().lines().count(), 2); + perms.insert("argus".to_owned(), pc2("false")); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + } + + #[tokio::test] + async fn an_answer_is_kept_for_ten_minutes_then_asked_again() { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + rig.dns.set("muede-pc2", &["192.168.23.50"]); + rig.pass(&perms, Duration::from_secs(599)) + .await + .expect("renders"); + assert_eq!(rig.dns.calls(), 1, "not asked before the ten minutes"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + rig.pass(&perms, RESOLVE_EVERY).await.expect("renders"); + assert_eq!(rig.dns.calls(), 2); + assert_eq!(rig.text(), "atlas 192.168.23.50 2222\n"); + } + + #[tokio::test] + async fn a_host_a_record_change_names_is_resolved_at_once() { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + rig.dns.set("other", &["192.168.23.9"]); + let mut perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + perms.insert( + "atlas".to_owned(), + record( + r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"other","port":22}]}"#, + ), + ); + rig.pass(&perms, Duration::from_secs(30)) + .await + .expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.9 22\n"); + assert_eq!( + rig.answers.by_host.keys().collect::>(), + ["other"], + "the answer of a host no record names is dropped" + ); + } + + #[tokio::test] + async fn a_failing_resolver_keeps_the_last_answer_and_asks_again_next_pass() { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + rig.dns.fail("muede-pc2"); + rig.pass(&perms, RESOLVE_EVERY).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n", "last good kept"); + // The failed ask is retried at the next pass, not in ten minutes. + let calls = rig.dns.calls(); + rig.pass(&perms, RESOLVE_EVERY + INTERVAL) + .await + .expect("renders"); + assert_eq!(rig.dns.calls(), calls + 1); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + // And it recovers. + rig.dns.set("muede-pc2", &["192.168.23.60"]); + rig.pass(&perms, RESOLVE_EVERY + INTERVAL * 2) + .await + .expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.60 2222\n"); + } + + #[tokio::test] + async fn a_name_that_never_resolved_allows_nothing() { + let mut rig = Rig::new(); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), ""); + } + + #[tokio::test] + async fn a_failure_is_warned_once_until_the_host_resolves_again() { + let mut rig = Rig::new(); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + for n in 0..3 { + rig.pass(&perms, INTERVAL * n).await.expect("renders"); + } + assert_eq!( + rig.answers.warned.iter().collect::>(), + ["muede-pc2"], + "one entry however many passes failed" + ); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + rig.pass(&perms, INTERVAL * 3).await.expect("renders"); + assert!(rig.answers.warned.is_empty(), "armed again by a success"); + } + + #[tokio::test] + async fn addresses_no_remote_machine_has_and_ipv6_are_dropped() { + let mut rig = Rig::new(); + rig.dns.set( + "muede-pc2", + &[ + "127.0.0.1", + "169.254.1.1", + "0.0.0.0", + "224.0.0.1", + "255.255.255.255", + "fd00::1", + "192.168.23.232", + ], + ); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + } + + #[tokio::test] + async fn an_agent_name_that_would_break_the_line_is_skipped() { + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([ + ("bad name\n".to_owned(), pc2("true")), + ("atlas".to_owned(), pc2("true")), + ]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + } + + #[tokio::test] + async fn an_unchanged_pass_leaves_the_file_alone_and_it_is_group_readable() { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let mut rig = Rig::new(); + rig.dns.set("muede-pc2", &["192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + let meta = std::fs::metadata(rig.file()).expect("stat"); + assert_eq!(meta.permissions().mode() & 0o777, 0o640); + rig.pass(&perms, INTERVAL).await.expect("renders"); + assert_eq!( + std::fs::metadata(rig.file()).expect("stat").ino(), + meta.ino(), + "nothing was written" + ); + } + fn two_hosts(first: &str, second: &str) -> permissions::Stored { + permissions::Stored::from([( + "atlas".to_owned(), + record(&format!( + r#"{{"ssh":true,"ssh_connections":[{{"name":"a","user":"root","host":"{first}","port":22}},{{"name":"b","user":"root","host":"{second}","port":2222}}]}}"# + )), + )]) + } + + /// A lookup that never answers counts as failed after the timeout: the + /// file is still written, with the other host's addresses, and the hung + /// host keeps the answer it had. + #[tokio::test(start_paused = true)] + async fn a_lookup_that_never_answers_does_not_hold_up_the_file() { + let mut rig = Rig::new(); + rig.dns.set("fast", &["192.168.23.1"]); + rig.dns.set("slow", &["192.168.23.2"]); + let perms = two_hosts("fast", "slow"); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!( + rig.text(), + "atlas 192.168.23.1 22\natlas 192.168.23.2 2222\n" + ); + // `slow` is due again and now hangs; `fast` changes its address. + rig.dns.hang("slow"); + rig.dns.set("fast", &["192.168.23.9"]); + let began = tokio::time::Instant::now(); + rig.pass(&perms, RESOLVE_EVERY).await.expect("renders"); + assert!( + began.elapsed() <= RESOLVE_TIMEOUT, + "bounded by one timeout: {:?}", + began.elapsed() + ); + assert_eq!( + rig.text(), + "atlas 192.168.23.2 2222\natlas 192.168.23.9 22\n", + "the other host's change landed and the hung host kept its answer" + ); + assert_eq!(rig.answers.warned.len(), 1, "one warning for the timeout"); + // A host that has never answered gets nothing, and the rest still land. + let mut fresh = Rig::new(); + fresh.dns.set("fast", &["192.168.23.1"]); + fresh.dns.hang("slow"); + fresh.pass(&perms, T0).await.expect("renders"); + assert_eq!(fresh.text(), "atlas 192.168.23.1 22\n"); + } + + /// Several hung hosts cost one timeout together, not one each. + #[tokio::test(start_paused = true)] + async fn hung_lookups_run_together() { + let mut rig = Rig::new(); + let hosts: Vec = (0..6).map(|n| format!("h{n}")).collect(); + for h in &hosts { + rig.dns.hang(h); + } + let conns: Vec = hosts + .iter() + .enumerate() + .map(|(n, h)| format!(r#"{{"name":"c{n}","user":"root","host":"{h}","port":22}}"#)) + .collect(); + let perms = permissions::Stored::from([( + "atlas".to_owned(), + record(&format!( + r#"{{"ssh":true,"ssh_connections":[{}]}}"#, + conns.join(",") + )), + )]); + let began = tokio::time::Instant::now(); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(began.elapsed(), RESOLVE_TIMEOUT); + assert_eq!(rig.dns.calls(), 6); + } + + #[test] + fn a_subnet_is_read_from_the_bridge_address_and_prefix() { + let net = Subnet::parse("10.42.0.1/24").expect("parses"); + assert!(net.contains("10.42.0.200".parse().unwrap())); + assert!(net.contains("10.42.0.1".parse().unwrap())); + assert!(!net.contains("10.42.1.1".parse().unwrap())); + let wide = Subnet::parse("10.42.5.9/15").expect("parses"); + assert!(wide.contains("10.43.255.1".parse().unwrap())); + assert!(!wide.contains("10.44.0.1".parse().unwrap())); + assert!( + Subnet::parse("0.0.0.0/0") + .expect("parses") + .contains("8.8.8.8".parse().unwrap()) + ); + for bad in ["10.42.0.1", "10.42.0.1/33", "nope/24", "10.42.0.1/x", ""] { + assert_eq!(Subnet::parse(bad), None, "{bad:?}"); + } + } + + #[tokio::test] + async fn an_address_inside_the_bridge_subnet_is_dropped_and_warned_once() { + let mut rig = Rig::new(); + rig.bridge = Subnet::parse("10.42.0.1/24"); + rig.dns.set("muede-pc2", &["10.42.0.7", "192.168.23.232"]); + let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n"); + assert!(rig.answers.warned_bridge.contains("muede-pc2")); + // Control: without the subnet the same answer allows both. + let mut open = Rig::new(); + open.dns.set("muede-pc2", &["10.42.0.7", "192.168.23.232"]); + open.pass(&perms, T0).await.expect("renders"); + assert_eq!( + open.text(), + "atlas 10.42.0.7 2222\natlas 192.168.23.232 2222\n" + ); + assert!(open.answers.warned_bridge.is_empty()); + // The warning is armed again once the host leaves the subnet. + rig.dns.set("muede-pc2", &["192.168.23.232"]); + rig.pass(&perms, RESOLVE_EVERY).await.expect("renders"); + assert!(rig.answers.warned_bridge.is_empty()); + } + + #[tokio::test] + async fn an_ip_literal_inside_the_bridge_subnet_allows_nothing() { + let mut rig = Rig::new(); + rig.bridge = Subnet::parse("10.42.0.1/24"); + let perms = permissions::Stored::from([( + "atlas".to_owned(), + record( + r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"10.42.0.9","port":22}]}"#, + ), + )]); + rig.pass(&perms, T0).await.expect("renders"); + assert_eq!(rig.text(), ""); + } +} diff --git a/hive-c0re/src/workers/mod.rs b/hive-c0re/src/workers/mod.rs index 003d4a387..2b691d24d 100644 --- a/hive-c0re/src/workers/mod.rs +++ b/hive-c0re/src/workers/mod.rs @@ -1,13 +1,15 @@ //! Background tasks and periodic sweeps: crash/login watcher, boot-time //! auto-update reconcile, the periodic declared ⇒ running reconcile tick, the agent-sockets.json writer loop, the MCP socket listener //! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the -//! swarm resource-cap and permissions reads, and the squid users file. +//! swarm resource-cap and permissions reads, the squid users file, and the +//! egress allowance for agents' SSH connections. //! Each submodule is re-exported //! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged. pub mod agent_sockets; pub mod auto_update; pub mod crash_watch; +pub mod egress_connections; pub mod knowledge; pub mod mcp_sockets; pub mod permissions_import; diff --git a/hive-priv/src/egress.rs b/hive-priv/src/egress.rs index 61e2cc988..576f9f29f 100644 --- a/hive-priv/src/egress.rs +++ b/hive-priv/src/egress.rs @@ -99,6 +99,12 @@ const LOCK: &str = "/run/lock/hive-egress-pin.lock"; /// `HIVE_EGRESS_UNRESTRICTED` names. Unset or unreadable lists no agent. const UNRESTRICTED_ENV: &str = "HIVE_EGRESS_UNRESTRICTED"; +/// The file of the agents' allowed connections, one ` ` per +/// line, that `HIVE_EGRESS_CONNECTIONS` names. Unset leaves the `agent_allow` set +/// alone: a ruleset that does not declare it still loads. Set but unreadable +/// lists no connection. +const CONNECTIONS_ENV: &str = "HIVE_EGRESS_CONNECTIONS"; + /// The prefix of an agent container's machine name. const AGENT_PREFIX: &str = "h-"; @@ -315,9 +321,66 @@ pub(crate) fn direct_macs(pins: &[Pin], unrestricted: &[String]) -> Vec { macs } +/// One connection an agent may open: its name, the address and the TCP port. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Connection { + pub(crate) agent: String, + pub(crate) ip: std::net::Ipv4Addr, + pub(crate) port: u16, +} + +/// The connections in `text`, one ` ` per line. A line of +/// another shape, a port 0 or a name that is no identifier is skipped. +pub(crate) fn parse_connections(text: &str) -> Vec { + text.lines() + .filter_map(|line| { + let mut cols = line.split_whitespace(); + let (agent, ip, port) = (cols.next()?, cols.next()?, cols.next()?); + if cols.next().is_some() { + return None; + } + hive_types::Ident::parse(agent).ok()?; + Some(Connection { + agent: agent.to_owned(), + ip: ip.parse().ok()?, + port: port.parse().ok().filter(|p| *p != 0)?, + }) + }) + .collect() +} + +/// The `agent_allow` elements of `connections`: for each, one per MAC bound to +/// a port of the agent's container (`h-`). An agent with no binding has +/// none, so nothing is allowed for it until its port is bound. +pub(crate) fn connection_elements(pins: &[Pin], connections: &[Connection]) -> String { + let mut rows: Vec<(Mac, std::net::Ipv4Addr, u16)> = Vec::new(); + for c in connections { + for p in pins { + if p.machine.strip_prefix(AGENT_PREFIX) == Some(c.agent.as_str()) { + rows.push((p.mac, c.ip, c.port)); + } + } + } + rows.sort_by_key(|(m, ip, port)| (m.0, *ip, *port)); + rows.dedup(); + let mut script = String::new(); + for (mac, ip, port) in rows { + let _ = writeln!( + script, + "add element {INET_TABLE} agent_allow {{ {mac} . {ip} . {port} }}" + ); + } + script +} + /// The nft input that adds the bindings `pins` to the two sets of the bridge -/// table and the [`direct_macs`] to the exemption set of the inet table. -pub(crate) fn pin_elements(pins: &[Pin], unrestricted: &[String]) -> String { +/// table, the [`direct_macs`] to the exemption set of the inet table and, when +/// `connections` is given, its [`connection_elements`]. +pub(crate) fn pin_elements( + pins: &[Pin], + unrestricted: &[String], + connections: Option<&[Connection]>, +) -> String { let mut script = String::new(); for Pin { port, mac, .. } in pins { let _ = writeln!( @@ -330,17 +393,30 @@ pub(crate) fn pin_elements(pins: &[Pin], unrestricted: &[String]) -> String { for mac in direct_macs(pins, unrestricted) { let _ = writeln!(script, "add element {INET_TABLE} direct_mac {{ {mac} }}"); } + if let Some(connections) = connections { + script.push_str(&connection_elements(pins, connections)); + } script } -/// The nft input that makes the bindings and the exemptions exactly those of -/// `pins` and `unrestricted`, in one transaction: a reader never sees a set -/// empty in between. -pub(crate) fn pin_script(pins: &[Pin], unrestricted: &[String]) -> String { +/// The nft input that makes the bindings, the exemptions and, when +/// `connections` is given, the allowed connections exactly those of `pins`, +/// `unrestricted` and `connections`, in one transaction: a reader never sees a +/// set empty in between. +pub(crate) fn pin_script( + pins: &[Pin], + unrestricted: &[String], + connections: Option<&[Connection]>, +) -> String { + let flush_connections = if connections.is_some() { + format!("flush set {INET_TABLE} agent_allow\n") + } else { + String::new() + }; format!( "flush set {TABLE} port_mac\nflush set {TABLE} port_chaddr\n\ - flush set {INET_TABLE} direct_mac\n{}", - pin_elements(pins, unrestricted) + flush set {INET_TABLE} direct_mac\n{flush_connections}{}", + pin_elements(pins, unrestricted, connections) ) } @@ -360,6 +436,15 @@ fn unrestricted_names() -> Vec { .collect() } +/// The connections in the file `HIVE_EGRESS_CONNECTIONS` names, or `None` when +/// it names none. A file that cannot be read lists no connection. +fn connection_list() -> Option> { + let path = std::env::var_os(CONNECTIONS_ENV).filter(|p| !p.is_empty())?; + Some(parse_connections( + &std::fs::read_to_string(path).unwrap_or_default(), + )) +} + /// The agent whose container holds the IPv4 address `ip`, as dnsmasq's lease /// file and the bindings of the ports say: the lease names a MAC, the MAC is /// bound to one port, and that port's machine is `h-`. An address in the @@ -900,7 +985,11 @@ fn pin(args: &[String]) -> Result<()> { let survey = { let _lock = lock(deadline)?; let survey = survey(&bridge)?; - nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?; + nft(&pin_script( + &survey.resolution.pins, + &unrestricted_names(), + connection_list().as_deref(), + ))?; commit(&survey)?; survey }; @@ -964,7 +1053,11 @@ fn load(args: &[String]) -> Result<()> { let _lock = lock(Instant::now() + WAIT)?; let found = survey(&bridge); let pins = match &found { - Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()), + Ok(found) => pin_elements( + &found.resolution.pins, + &unrestricted_names(), + connection_list().as_deref(), + ), Err(e) => { eprintln!("{LOAD}: binding the ports failed, loading without bindings: {e:#}"); String::new() @@ -1241,7 +1334,7 @@ mod tests { #[test] fn the_ruleset_loads_with_the_bindings_and_leases_in_one_script() { let calls = std::cell::RefCell::new(Vec::new()); - let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]); + let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[], None); let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap(); load_ruleset("table bridge hive_egress { }", &pins, &leases, |s| { calls.borrow_mut().push(s.to_string()); @@ -1260,7 +1353,7 @@ mod tests { #[test] fn a_lease_load_that_fails_is_a_warning_and_the_bindings_still_load() { let calls = std::cell::RefCell::new(Vec::new()); - let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]); + let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[], None); let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap(); let result = load_ruleset("RULESET", &pins, &leases, |s| { calls.borrow_mut().push(s.to_string()); @@ -1284,7 +1377,7 @@ mod tests { #[test] fn pin_script_replaces_all_three_sets_in_one_script() { - let script = pin_script(&[pin("vb-h-a", 10, 10, "h-a")], &[]); + let script = pin_script(&[pin("vb-h-a", 10, 10, "h-a")], &[], None); assert_eq!( script, "flush set bridge hive_egress port_mac\n\ @@ -1293,7 +1386,7 @@ mod tests { add element bridge hive_egress port_mac { \"vb-h-a\" . 02:00:00:00:00:0a }\n\ add element bridge hive_egress port_chaddr { \"vb-h-a\" . 0x02000000000a }\n" ); - assert!(!pin_script(&[], &[]).contains("add element")); + assert!(!pin_script(&[], &[], None).contains("add element")); } /// Service containers and the agents the file lists are exempt; an agent the @@ -1310,13 +1403,102 @@ mod tests { assert_eq!(direct_macs(&pins, &listed), vec![mac(2), mac(3)]); // Control: with nothing listed, only the service container stays. assert_eq!(direct_macs(&pins, &[]), vec![mac(3)]); - let script = pin_elements(&pins, &listed); + let script = pin_elements(&pins, &listed, None); assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:02 }")); assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:03 }")); assert!(!script.contains("direct_mac { 02:00:00:00:00:01 }")); assert!(!script.contains("direct_mac { 02:00:00:00:00:04 }")); } + fn conn(agent: &str, ip: &str, port: u16) -> Connection { + Connection { + agent: agent.into(), + ip: ip.parse().unwrap(), + port, + } + } + + /// An agent's connection is allowed from the MAC of that agent's port only, + /// not from a container whose name merely starts the same or from a service + /// container. + #[test] + fn a_connection_is_allowed_from_its_agents_mac_only() { + let pins = [ + pin("vb-h-atlas", 10, 1, "h-atlas"), + pin("vb-h-atlasx", 11, 2, "h-atlasx"), + pin("vb-matrix", 12, 3, "atlas"), + ]; + let script = connection_elements(&pins, &[conn("atlas", "192.168.23.232", 2222)]); + assert_eq!( + script, + "add element inet hive_egress agent_allow \ + { 02:00:00:00:00:01 . 192.168.23.232 . 2222 }\n" + ); + } + + #[test] + fn a_connection_of_an_unbound_agent_adds_nothing_and_duplicates_collapse() { + let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")]; + assert_eq!( + connection_elements(&pins, &[conn("uma", "10.0.0.1", 22)]), + "" + ); + let twice = [conn("atlas", "10.0.0.1", 22), conn("atlas", "10.0.0.1", 22)]; + assert_eq!(connection_elements(&pins, &twice).lines().count(), 1); + assert_eq!(connection_elements(&[], &twice), ""); + } + + #[test] + fn the_connections_file_is_parsed_line_by_line_and_bad_lines_are_skipped() { + let text = "atlas 192.168.23.232 2222\n\ + argus 10.0.0.1 22\n\ + \n\ + atlas not-an-ip 22\n\ + atlas 10.0.0.2 0\n\ + atlas 10.0.0.2 70000\n\ + bad;name 10.0.0.3 22\n\ + atlas 10.0.0.4 22 extra\n\ + atlas 10.0.0.5\n"; + assert_eq!( + parse_connections(text), + [ + conn("atlas", "192.168.23.232", 2222), + conn("argus", "10.0.0.1", 22) + ] + ); + } + + /// With a connections file the script flushes `agent_allow` in the same + /// transaction that refills it; without one it does not name the set, so a + /// ruleset that never declared it still loads. + #[test] + fn the_script_flushes_and_fills_agent_allow_only_when_a_file_is_named() { + let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")]; + let with = pin_script(&pins, &[], Some(&[conn("atlas", "10.0.0.1", 22)])); + let flush = with + .find("flush set inet hive_egress agent_allow\n") + .expect("flushed"); + let add = with + .find("add element inet hive_egress agent_allow { 02:00:00:00:00:01 . 10.0.0.1 . 22 }") + .expect("added"); + assert!(flush < add, "the flush comes first"); + // Control: a named file with no connection still flushes the old ones. + let empty = pin_script(&pins, &[], Some(&[])); + assert!(empty.contains("flush set inet hive_egress agent_allow\n")); + assert!(!empty.contains("add element inet hive_egress agent_allow")); + let none = pin_script(&pins, &[], None); + assert!(!none.contains("agent_allow"), "{none}"); + assert!(none.contains("flush set inet hive_egress direct_mac")); + } + + #[test] + fn the_load_script_fills_agent_allow_without_flushing_it() { + let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")]; + let script = pin_elements(&pins, &[], Some(&[conn("atlas", "10.0.0.1", 22)])); + assert!(script.contains("agent_allow { 02:00:00:00:00:01 . 10.0.0.1 . 22 }")); + assert!(!script.contains("flush")); + } + const NOW: u64 = 1_000; fn lease_line(expiry: u64, last: u8, ip: &str) -> String { diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 2a006445b..5102eadaf 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -298,6 +298,14 @@ in HIVE_SQUID_PROXY = config.services.hyperhive.deploy.hive-controller.squid.listen; HIVE_EGRESS_NO_PROXY = lib.concatStringsSep "," egressDirectHosts; } +// + lib.optionalAttrs (config.services.hyperhive.deploy.hive-controller.egress.connectionsFile != null) + { + # Where `workers::egress_connections` renders the SSH connections each agent + # may open, for the egress forward chain. + HIVE_EGRESS_CONNECTIONS_FILE = + config.services.hyperhive.deploy.hive-controller.egress.connectionsFile; + } // lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) { # Absent means the system trust store — right for a deployment with a real # CA, wrong for the self-signed one ../glue-bao-tls.nix mints, which is why diff --git a/nix/host-modules/hive-egress.nix b/nix/host-modules/hive-egress.nix index 960f098b3..9c850c84f 100644 --- a/nix/host-modules/hive-egress.nix +++ b/nix/host-modules/hive-egress.nix @@ -74,6 +74,9 @@ let } // lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) { HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile; + } + // lib.optionalAttrs (egressCfg.connectionsFile != null) { + HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile; }; leaseScript = pkgs.writeShellScript "hive-egress-lease" '' export PATH=${lib.makeBinPath [ pkgs.nftables ]} @@ -186,6 +189,7 @@ let chain forward { type filter hook forward priority filter - 10; policy accept; ${fromBridge} ether saddr @direct_mac accept + ${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept ${fromBridge} counter ${lib.concatStringsSep "\n " ( logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules @@ -309,6 +313,7 @@ let ${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''} } set direct_mac { type ether_addr; } + set agent_allow { type ether_addr . ipv4_addr . inet_service; } ${lib.optionalString egressCfg.intercept.enable interceptChain} ${lib.optionalString (egressCfg.forward != "accept") forwardChain} } @@ -342,7 +347,9 @@ in What the `inet hive_egress` table does with traffic the host forwards from the agent bridge to anywhere but the bridge. In `log` and `reject` the traffic of the MACs in the `direct_mac` set (the unrestricted - agents and the service containers) is accepted and never logged. + agents and the service containers) is accepted and never logged, and so + is TCP from an agent to an address and port in the `agent_allow` set + (`connectionsFile`). - `accept`: nothing; the table has no forward chain. - `log`: each new flow is logged with the prefix `hive-egress-would-drop `, @@ -359,6 +366,22 @@ in ''; }; + options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The file hive-c0re renders with the SSH connections each agent may open, + one ` ` per line: the addresses behind the `host` of + each connection in the permissions record of an agent whose `ssh` switch + is on. `hive-priv egress-pin` turns each line into elements of the + `agent_allow` set of the `inet hive_egress` table, keyed by the MAC of + the agent's port, and the forward chain accepts TCP to an address and + port in it before it logs or refuses. The file is read when a port is + bound, on each pass of the 30 second timer and when the file changes. + `null` leaves the set alone, empty: no agent has an allowance. + ''; + }; + options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption { type = lib.types.enum [ "drop" @@ -664,6 +687,17 @@ in }; }; + # The same pass refills `agent_allow`, so a connection an operator declares + # is allowed at once. + systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) { + description = "Watch the agents' allowed connections list for the egress forward chain"; + wantedBy = [ "paths.target" ]; + pathConfig = { + PathChanged = toString egressCfg.connectionsFile; + Unit = "hive-egress-pin.service"; + }; + }; + # nspawn names the host end of a container's veth `vb-` when the # container joins a bridge. services.udev.extraRules = '' diff --git a/nix/host-modules/hive-squid.nix b/nix/host-modules/hive-squid.nix index f4c3c24d5..1d5887168 100644 --- a/nix/host-modules/hive-squid.nix +++ b/nix/host-modules/hive-squid.nix @@ -43,6 +43,7 @@ let proxyUrl = "http://${netCfg.bridgeIp}:${toString port}"; exporterListen = "127.0.0.1:9301"; interceptCfg = hyperhiveCfg.deploy.hive-controller.egress.intercept; + egressCfg = hyperhiveCfg.deploy.hive-controller.egress; # The path every platform container's unit runs hive-priv by (hive-priv.nix # points it at the current build). Squid runs `egress-whois` through it. hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv; @@ -555,6 +556,11 @@ in enable = lib.mkDefault true; inherit (cfg) unrestrictedFile agentAllowFile; }; + # The file `hive-c0re` renders with the SSH connections each agent may open, + # which the egress forward chain reads (`workers::egress_connections`). It + # sits in the directory hive-c0re may write. + services.hyperhive.deploy.hive-controller.egress.connectionsFile = + lib.mkDefault "${usersDir}/ssh-connections"; services.hyperhive.network.enable = lib.mkDefault true; services.hyperhive.otel.scrapeTargets.squid = exporterListen; @@ -591,7 +597,10 @@ in "C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}" "f ${cfg.agentAllowFile} 0640 hive-core squid - -" "f ${cfg.denyFile} 0640 hive-core squid - -" - ]; + ] + ++ lib.optional ( + egressCfg.connectionsFile != null + ) "f ${egressCfg.connectionsFile} 0640 hive-core squid - -"; # Squid reads the unrestricted list, the allow-list and the per-agent include # when it parses its configuration, not when they change. hive-c0re writes diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index 8f740db45..894ae8853 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -1645,6 +1645,57 @@ let && !(lib.any (l: lib.hasSuffix " drop" l || lib.hasSuffix " reject" l) rules) && lib.hasInfix "chain input" text; } + { + # An agent's declared SSH connections are accepted before the chain + # counts, logs or refuses, in `log` and in `reject`. The set takes MAC, + # address and port. The host without a controller has the set and no + # chain (the presence control) and no wiring of the connections file. + # hive-c0re, the units that fill the sets and the path unit that runs a + # pass are told the same file. + name = "the egress forward chain accepts the agents' declared ssh connections before it logs or refuses"; + ok = + let + rulesetOf = m: m.services.hyperhive.network.egress.ruleset; + inetOf = m: lib.head (lib.splitString "table bridge hive_egress {" (rulesetOf m)); + rulesOf = + m: + lib.filter (l: l != "" && l != "}") ( + map lib.trim (lib.splitString "\n" (lib.last (lib.splitString "chain forward" (inetOf m)))) + ); + idxIn = rules: pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules; + allow = ''iifname "hive-br0" oifname != "hive-br0" meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept''; + set = "set agent_allow { type ether_addr . ipv4_addr . inet_service; }"; + inOrder = + m: + let + rules = rulesOf m; + idx = idxIn rules; + in + lib.elem allow rules + && lib.length (lib.filter (r: r == allow) rules) == 1 + && idx "ether saddr @direct_mac accept" < idx "@agent_allow accept" + && idx "@agent_allow accept" < idx "counter" + && idx "@agent_allow accept" < idx "update @log_v4" + && lib.hasInfix set (inetOf m); + file = "/var/lib/hive-squid/ssh-connections"; + envOf = m: name: m.systemd.services.${name}.environment; + in + inOrder bare + && inOrder egressForwardReject + && lib.hasInfix set (inetOf egressForwardAccept) + && !(lib.hasInfix "@agent_allow accept" (rulesetOf egressForwardAccept)) + && bare.services.hyperhive.deploy.hive-controller.egress.connectionsFile == file + && bare.systemd.services.hive-c0re.environment.HIVE_EGRESS_CONNECTIONS_FILE == file + && lib.all (name: ((envOf bare name).HIVE_EGRESS_CONNECTIONS or null) == file) [ + "hive-egress" + "hive-egress-pin@" + "hive-egress-pin" + ] + && bare.systemd.paths.hive-egress-connections.pathConfig.PathChanged == file + && bare.systemd.paths.hive-egress-connections.pathConfig.Unit == "hive-egress-pin.service" + && !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_CONNECTIONS) + && !(egressForwardAccept.systemd.paths ? hive-egress-connections); + } { # The bridge table drops what a port sends from an address that is not # its own, and logs it. The ruleset is read as text: each `spoof_` @@ -1807,7 +1858,7 @@ let chain = lib.last (lib.splitString "chain forward" text); rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain)); idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules; - tcp = lib.filter (lib.hasInfix " meta l4proto tcp ") rules; + tcp = lib.filter (l: lib.hasInfix " meta l4proto tcp " l && !(lib.hasInfix "@agent_allow" l)) rules; other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules; logged = lib.filter (lib.hasInfix "log prefix") rules; bareText = bare.services.hyperhive.network.egress.ruleset; @@ -1820,7 +1871,7 @@ let && !(lib.hasInfix "wg-hive" text) && idx "ether saddr @direct_mac accept" < idx "counter" && idx "counter" < idx "reject" - && lib.length (lib.filter (lib.hasSuffix " accept") rules) == 1 + && lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2 && lib.length tcp == 4 && lib.length other == 4 && lib.all (lib.hasSuffix " reject with tcp reset") tcp