Watch
0
0
Fork
You've already forked hyperhive
0

egress: the forward chain allows each agent's declared ssh connections

This commit is contained in:
atlas 2026-11-20 03:07:14 +01:00
commit 8ece0b7e0b
8 changed files with 1110 additions and 20 deletions

View file

@ -321,6 +321,8 @@ async fn cmd_serve(
swarm_agent_status::spawn(std::sync::Arc::clone(&coord));
// Squid's users file: the hashes of this hive's agents' proxy credentials.
workers::squid_users::spawn(coord.shutdown_rx());
// The SSH connections the forward chain lets each agent open.
workers::egress_connections::spawn(coord.shutdown_rx());
// Per-agent bash-tasks file cleanup, and the removal of the old event log,
// now run agent-side in the harness (`hive_agent::vacuum`): the files are
// agent-owned, so host-side deletes hit PermissionDenied / readonly-db

View file

@ -0,0 +1,802 @@
//! The SSH connections the hive's forward chain lets agents open: for each
//! agent whose `ssh` switch is on, the addresses behind the `host` of each
//! connection in its permissions record.
//!
//! Every [`INTERVAL`] this reads the permissions the swarm granted this
//! hive's agents ([`crate::permissions`]) and renders one `<agent> <ip>
//! <port>` line per agent, IPv4 address and port into the file named by
//! [`ENV_FILE`]. `hive-priv egress-pin` turns each line into one element of
//! the `agent_allow` set of `nix/host-modules/hive-egress.nix`, keyed by the
//! MAC of the agent's port, which only hive-priv can tell. IPv6 answers are
//! left out: the agent bridge has no IPv6 addressing.
//!
//! A host name is resolved by the system resolver, so it means what it means
//! to the host's other services; an IPv4 literal is taken as it is. Every
//! address a name resolves to is allowed, as the agent may use any of them.
//! An answer is kept for [`RESOLVE_EVERY`], and a host a record newly names
//! is resolved at the next pass. The hosts of a pass are asked concurrently,
//! each for at most [`RESOLVE_TIMEOUT`], so a resolver that hangs delays the
//! file by that long and no more. A name that fails to resolve, or times out,
//! keeps the addresses of its last answer and logs one warning until it
//! resolves again: a resolver outage must not read as the operator removing
//! the connection. An address inside this hive's bridge subnet is dropped,
//! with one warning per host: a flow to it never reaches the forward chain.
//!
//! A failed read of the permissions leaves the file as it is.
use std::collections::{BTreeSet, HashMap, HashSet};
use std::future::Future;
use std::net::{IpAddr, Ipv4Addr};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use futures_util::StreamExt as _;
use super::squid_users::write_if_changed;
use crate::permissions;
use crate::stats::sweep_health::{self, SweepHealth};
/// How often the file is re-rendered, which is how soon a new connection
/// or a flipped `ssh` switch takes effect.
pub const INTERVAL: Duration = Duration::from_secs(30);
/// How long an answer of the resolver is used before the name is asked again.
pub const RESOLVE_EVERY: Duration = Duration::from_mins(10);
/// How long one lookup may take before it counts as failed.
pub const RESOLVE_TIMEOUT: Duration = Duration::from_secs(5);
/// How many lookups of one pass run at the same time.
const CONCURRENT_LOOKUPS: usize = 8;
/// At [`INTERVAL`] this is ~90 seconds of failing passes.
const FAILURES_BEFORE_WARNING: u32 = 3;
/// The environment variable naming the file. Set by `nix/host-modules/hive-c0re`
/// from `services.hyperhive.deploy.hive-controller.egress.connectionsFile`;
/// unset means this host's forward chain does not read one.
pub const ENV_FILE: &str = "HIVE_EGRESS_CONNECTIONS_FILE";
/// The environment variable that carries the bridge address and prefix
/// (`10.42.0.1/24`); `nix/host-modules/hive-network.nix` sets it for the daemon.
const ENV_SUBNET: &str = "HIVE_NETWORK_SUBNET";
/// An IPv4 subnet.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Subnet {
network: u32,
mask: u32,
}
impl Subnet {
/// `<ipv4>/<prefix>`, where the address may be any host of the subnet.
fn parse(text: &str) -> Option<Self> {
let (ip, prefix) = text.split_once('/')?;
let prefix: u32 = prefix.parse().ok().filter(|p| *p <= 32)?;
let mask = u32::MAX.checked_shl(32 - prefix).unwrap_or(0);
Some(Self {
network: u32::from(ip.parse::<Ipv4Addr>().ok()?) & mask,
mask,
})
}
fn contains(self, ip: Ipv4Addr) -> bool {
u32::from(ip) & self.mask == self.network
}
}
/// One declared connection of an agent: where it goes.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct Target {
agent: String,
host: String,
port: u16,
}
/// The connections of every agent whose `ssh` switch is on.
fn targets(perms: &permissions::Stored) -> BTreeSet<Target> {
let mut out = BTreeSet::new();
for (agent, record) in perms {
if record.ssh != Some(true) || hive_types::Ident::parse(agent).is_err() {
continue;
}
for c in &record.ssh_connections {
out.insert(Target {
agent: agent.clone(),
host: c.host.clone(),
port: c.port,
});
}
}
out
}
/// What a host resolved to at its last successful ask.
struct Answer {
/// When it was asked; `None` once an ask failed, so the next pass asks
/// again.
at: Option<Instant>,
addrs: Vec<Ipv4Addr>,
}
/// What each host resolved to.
#[derive(Default)]
struct Answers {
by_host: HashMap<String, Answer>,
/// Hosts whose failure was logged and that have not resolved since.
warned: HashSet<String>,
/// Hosts whose answer holds an address of the bridge subnet, logged once.
warned_bridge: HashSet<String>,
}
impl Answers {
/// Ask for every host in `hosts` that has no answer yet or whose answer is
/// older than [`RESOLVE_EVERY`], and forget the others. The asks run
/// [`CONCURRENT_LOOKUPS`] at a time, each for at most [`RESOLVE_TIMEOUT`].
async fn refresh<F, Fut>(
&mut self,
hosts: &BTreeSet<&str>,
now: Instant,
bridge: Option<Subnet>,
resolve: F,
) where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Vec<IpAddr>>>,
{
self.by_host.retain(|h, _| hosts.contains(h.as_str()));
self.warned.retain(|h| hosts.contains(h.as_str()));
self.warned_bridge.retain(|h| hosts.contains(h.as_str()));
let due: Vec<String> = hosts
.iter()
.filter(|host| {
self.by_host
.get(**host)
.and_then(|a| a.at)
.is_none_or(|at| now.duration_since(at) >= RESOLVE_EVERY)
})
.map(|host| (*host).to_owned())
.collect();
let resolve = &resolve;
let asked: Vec<(String, Result<Vec<IpAddr>>)> = futures_util::stream::iter(due)
.map(|host| async move {
let answer = ask(&host, resolve).await;
(host, answer)
})
.buffer_unordered(CONCURRENT_LOOKUPS)
.collect()
.await;
for (host, result) in asked {
let host = host.as_str();
match result {
Ok(addrs) => {
self.warned.remove(host);
let (usable, inside) = usable_v4(&addrs, bridge);
if inside.is_empty() {
self.warned_bridge.remove(host);
} else if self.warned_bridge.insert(host.to_owned()) {
tracing::warn!(
host,
addresses = ?inside,
"egress connections: the host resolves into the hive's bridge \
subnet; those addresses are not allowed"
);
}
self.by_host.insert(
host.to_owned(),
Answer {
at: Some(now),
addrs: usable,
},
);
}
Err(e) => {
if self.warned.insert(host.to_owned()) {
tracing::warn!(
host,
error = ?e,
"egress connections: resolving failed, keeping the last answer"
);
}
if let Some(answer) = self.by_host.get_mut(host) {
answer.at = None;
}
}
}
}
}
fn addrs(&self, host: &str) -> &[Ipv4Addr] {
self.by_host.get(host).map_or(&[], |a| a.addrs.as_slice())
}
}
/// `host` as an address when it is a literal, else what `resolve` answers
/// within [`RESOLVE_TIMEOUT`].
async fn ask<F, Fut>(host: &str, resolve: &F) -> Result<Vec<IpAddr>>
where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Vec<IpAddr>>>,
{
if let Ok(ip) = host.parse::<IpAddr>() {
return Ok(vec![ip]);
}
tokio::time::timeout(RESOLVE_TIMEOUT, resolve(host.to_owned()))
.await
.map_err(|_| anyhow::anyhow!("resolving {host} took more than {RESOLVE_TIMEOUT:?}"))?
}
/// The IPv4 addresses among `addrs` that a remote machine can have, and the
/// ones dropped for lying inside the `bridge` subnet.
fn usable_v4(addrs: &[IpAddr], bridge: Option<Subnet>) -> (Vec<Ipv4Addr>, Vec<Ipv4Addr>) {
let mut v4: Vec<Ipv4Addr> = addrs
.iter()
.filter_map(|a| match a {
IpAddr::V4(v4) => Some(*v4),
IpAddr::V6(_) => None,
})
.filter(|a| {
!(a.is_loopback()
|| a.is_link_local()
|| a.is_unspecified()
|| a.is_multicast()
|| a.is_broadcast())
})
.collect();
v4.sort_unstable();
v4.dedup();
v4.into_iter()
.partition(|a| !bridge.is_some_and(|b| b.contains(*a)))
}
/// The file's text for `targets` under the answers in `answers`, sorted.
fn render(targets: &BTreeSet<Target>, answers: &Answers) -> String {
let mut lines = BTreeSet::new();
for t in targets {
for ip in answers.addrs(&t.host) {
lines.insert(format!("{} {ip} {}\n", t.agent, t.port));
}
}
lines.into_iter().collect()
}
/// One pass: refresh the answers for the hosts `perms` names and write the file
/// if it changed.
async fn pass_with<F, Fut>(
perms: &permissions::Stored,
answers: &mut Answers,
file: &Path,
now: Instant,
bridge: Option<Subnet>,
resolve: F,
) -> Result<()>
where
F: Fn(String) -> Fut,
Fut: Future<Output = Result<Vec<IpAddr>>>,
{
let wanted = targets(perms);
let hosts: BTreeSet<&str> = wanted.iter().map(|t| t.host.as_str()).collect();
answers.refresh(&hosts, now, bridge, resolve).await;
let text = render(&wanted, answers);
if write_if_changed(file, &text)? {
tracing::info!(
file = %file.display(),
"egress connections: wrote the list, {} allowed addresses",
text.lines().count()
);
}
Ok(())
}
/// The system resolver's addresses for `host`.
async fn system_resolve(host: String) -> Result<Vec<IpAddr>> {
let found = tokio::net::lookup_host((host.as_str(), 0))
.await
.with_context(|| format!("resolving {host}"))?;
Ok(found.map(|a| a.ip()).collect())
}
fn env_path(name: &str) -> Option<PathBuf> {
std::env::var_os(name)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
}
/// Start the pass loop, when this host names the file.
pub fn spawn(mut shutdown: tokio::sync::watch::Receiver<bool>) {
let Some(file) = env_path(ENV_FILE) else {
return;
};
let bridge = std::env::var(ENV_SUBNET)
.ok()
.and_then(|s| Subnet::parse(&s));
if bridge.is_none() {
tracing::warn!(
"egress connections: {ENV_SUBNET} is unset or invalid, so the bridge subnet is not excluded"
);
}
tokio::spawn(async move {
let mut health =
SweepHealth::new("egress_connections_render", "warn", FAILURES_BEFORE_WARNING);
let mut answers = Answers::default();
loop {
let result = match permissions::read_at(&permissions::path()) {
Ok(perms) => {
pass_with(
&perms,
&mut answers,
&file,
Instant::now(),
bridge,
system_resolve,
)
.await
}
Err(e) => {
Err(anyhow::Error::new(e).context("reading the stored swarm permissions"))
}
};
match result {
Ok(()) => health.record_ok(),
Err(e) => {
tracing::warn!(error = ?e, "egress connections: the pass failed");
let err = format!("{e:#}");
health.record_err(|ctx| {
let age = ctx.since_last_ok.map_or_else(
|| "no success this session".to_owned(),
|d| format!("last ok {} ago", sweep_health::fmt_age(d)),
);
format!(
"rendering the agents' allowed ssh connections is failing ({} \
consecutive, {age}) — the forward chain keeps the list last \
written: {err}",
ctx.consecutive
)
});
}
}
tokio::select! {
() = tokio::time::sleep(INTERVAL) => {}
_ = shutdown.changed() => return,
}
}
});
}
#[cfg(test)]
mod tests {
use std::cell::RefCell;
use std::rc::Rc;
use super::*;
use swarm_wire::agent_permissions::AgentPermissions;
fn record(json: &str) -> AgentPermissions {
serde_json::from_str(json).expect("decodes")
}
fn pc2(agent_ssh: &str) -> AgentPermissions {
record(&format!(
r#"{{"ssh":{agent_ssh},"ssh_connections":[{{"name":"k8s-lab-vm","user":"root","host":"muede-pc2","port":2222}}]}}"#
))
}
fn ip(s: &str) -> IpAddr {
s.parse().expect("an address")
}
type ResolveFuture = std::pin::Pin<Box<dyn Future<Output = Result<Vec<IpAddr>>>>>;
/// A resolver that answers from `table`, counts its calls and fails for
/// names not in it.
#[derive(Clone, Default)]
struct Dns {
table: Rc<RefCell<HashMap<String, Vec<IpAddr>>>>,
calls: Rc<RefCell<Vec<String>>>,
/// Hosts whose lookup never answers.
hung: Rc<RefCell<HashSet<String>>>,
}
impl Dns {
fn set(&self, host: &str, addrs: &[&str]) {
self.table
.borrow_mut()
.insert(host.to_owned(), addrs.iter().map(|a| ip(a)).collect());
}
fn fail(&self, host: &str) {
self.table.borrow_mut().remove(host);
}
fn hang(&self, host: &str) {
self.hung.borrow_mut().insert(host.to_owned());
}
fn resolver(&self) -> impl Fn(String) -> ResolveFuture + use<> {
let (table, calls, hung) = (self.table.clone(), self.calls.clone(), self.hung.clone());
move |host| {
calls.borrow_mut().push(host.clone());
if hung.borrow().contains(&host) {
return Box::pin(std::future::pending());
}
let answer = table
.borrow()
.get(&host)
.cloned()
.ok_or_else(|| anyhow::anyhow!("no such host"));
Box::pin(std::future::ready(answer))
}
}
fn calls(&self) -> usize {
self.calls.borrow().len()
}
}
struct Rig {
dir: tempfile::TempDir,
answers: Answers,
dns: Dns,
start: Instant,
bridge: Option<Subnet>,
}
impl Rig {
fn new() -> Self {
let rig = Self {
dir: tempfile::tempdir().expect("tempdir"),
answers: Answers::default(),
dns: Dns::default(),
start: Instant::now(),
bridge: None,
};
// The empty file `systemd-tmpfiles` creates before c0re starts.
std::fs::write(rig.file(), "").expect("write");
rig
}
fn file(&self) -> PathBuf {
self.dir.path().join("connections")
}
async fn pass(&mut self, perms: &permissions::Stored, after: Duration) -> Result<()> {
let (file, now) = (self.file(), self.start + after);
pass_with(
perms,
&mut self.answers,
&file,
now,
self.bridge,
self.dns.resolver(),
)
.await
}
fn text(&self) -> String {
std::fs::read_to_string(self.file()).expect("the file is there")
}
}
const T0: Duration = Duration::ZERO;
#[tokio::test]
async fn a_name_is_allowed_at_every_address_it_resolves_to() {
let mut rig = Rig::new();
rig.dns
.set("muede-pc2", &["192.168.23.232", "192.168.23.183"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(
rig.text(),
"atlas 192.168.23.183 2222\natlas 192.168.23.232 2222\n"
);
}
#[tokio::test]
async fn an_ip_literal_passes_through_without_a_lookup() {
let mut rig = Rig::new();
let perms = permissions::Stored::from([(
"atlas".to_owned(),
record(
r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"10.1.2.3","port":22}]}"#,
),
)]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 10.1.2.3 22\n");
assert_eq!(rig.dns.calls(), 0);
}
#[tokio::test]
async fn the_ssh_switch_off_or_unset_allows_nothing() {
for ssh in ["false", "null"] {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2(ssh))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "", "ssh = {ssh}");
assert_eq!(rig.dns.calls(), 0, "a switched-off agent costs no lookup");
}
// Control: the same record with the switch on is allowed.
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
}
#[tokio::test]
async fn switching_ssh_off_removes_the_agents_lines_and_only_its_own() {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let mut perms = permissions::Stored::from([
("atlas".to_owned(), pc2("true")),
("argus".to_owned(), pc2("true")),
]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text().lines().count(), 2);
perms.insert("argus".to_owned(), pc2("false"));
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
}
#[tokio::test]
async fn an_answer_is_kept_for_ten_minutes_then_asked_again() {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
rig.dns.set("muede-pc2", &["192.168.23.50"]);
rig.pass(&perms, Duration::from_secs(599))
.await
.expect("renders");
assert_eq!(rig.dns.calls(), 1, "not asked before the ten minutes");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
rig.pass(&perms, RESOLVE_EVERY).await.expect("renders");
assert_eq!(rig.dns.calls(), 2);
assert_eq!(rig.text(), "atlas 192.168.23.50 2222\n");
}
#[tokio::test]
async fn a_host_a_record_change_names_is_resolved_at_once() {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
rig.dns.set("other", &["192.168.23.9"]);
let mut perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
perms.insert(
"atlas".to_owned(),
record(
r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"other","port":22}]}"#,
),
);
rig.pass(&perms, Duration::from_secs(30))
.await
.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.9 22\n");
assert_eq!(
rig.answers.by_host.keys().collect::<Vec<_>>(),
["other"],
"the answer of a host no record names is dropped"
);
}
#[tokio::test]
async fn a_failing_resolver_keeps_the_last_answer_and_asks_again_next_pass() {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
rig.dns.fail("muede-pc2");
rig.pass(&perms, RESOLVE_EVERY).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n", "last good kept");
// The failed ask is retried at the next pass, not in ten minutes.
let calls = rig.dns.calls();
rig.pass(&perms, RESOLVE_EVERY + INTERVAL)
.await
.expect("renders");
assert_eq!(rig.dns.calls(), calls + 1);
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
// And it recovers.
rig.dns.set("muede-pc2", &["192.168.23.60"]);
rig.pass(&perms, RESOLVE_EVERY + INTERVAL * 2)
.await
.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.60 2222\n");
}
#[tokio::test]
async fn a_name_that_never_resolved_allows_nothing() {
let mut rig = Rig::new();
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "");
}
#[tokio::test]
async fn a_failure_is_warned_once_until_the_host_resolves_again() {
let mut rig = Rig::new();
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
for n in 0..3 {
rig.pass(&perms, INTERVAL * n).await.expect("renders");
}
assert_eq!(
rig.answers.warned.iter().collect::<Vec<_>>(),
["muede-pc2"],
"one entry however many passes failed"
);
rig.dns.set("muede-pc2", &["192.168.23.232"]);
rig.pass(&perms, INTERVAL * 3).await.expect("renders");
assert!(rig.answers.warned.is_empty(), "armed again by a success");
}
#[tokio::test]
async fn addresses_no_remote_machine_has_and_ipv6_are_dropped() {
let mut rig = Rig::new();
rig.dns.set(
"muede-pc2",
&[
"127.0.0.1",
"169.254.1.1",
"0.0.0.0",
"224.0.0.1",
"255.255.255.255",
"fd00::1",
"192.168.23.232",
],
);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
}
#[tokio::test]
async fn an_agent_name_that_would_break_the_line_is_skipped() {
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([
("bad name\n".to_owned(), pc2("true")),
("atlas".to_owned(), pc2("true")),
]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
}
#[tokio::test]
async fn an_unchanged_pass_leaves_the_file_alone_and_it_is_group_readable() {
use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _};
let mut rig = Rig::new();
rig.dns.set("muede-pc2", &["192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
let meta = std::fs::metadata(rig.file()).expect("stat");
assert_eq!(meta.permissions().mode() & 0o777, 0o640);
rig.pass(&perms, INTERVAL).await.expect("renders");
assert_eq!(
std::fs::metadata(rig.file()).expect("stat").ino(),
meta.ino(),
"nothing was written"
);
}
fn two_hosts(first: &str, second: &str) -> permissions::Stored {
permissions::Stored::from([(
"atlas".to_owned(),
record(&format!(
r#"{{"ssh":true,"ssh_connections":[{{"name":"a","user":"root","host":"{first}","port":22}},{{"name":"b","user":"root","host":"{second}","port":2222}}]}}"#
)),
)])
}
/// A lookup that never answers counts as failed after the timeout: the
/// file is still written, with the other host's addresses, and the hung
/// host keeps the answer it had.
#[tokio::test(start_paused = true)]
async fn a_lookup_that_never_answers_does_not_hold_up_the_file() {
let mut rig = Rig::new();
rig.dns.set("fast", &["192.168.23.1"]);
rig.dns.set("slow", &["192.168.23.2"]);
let perms = two_hosts("fast", "slow");
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(
rig.text(),
"atlas 192.168.23.1 22\natlas 192.168.23.2 2222\n"
);
// `slow` is due again and now hangs; `fast` changes its address.
rig.dns.hang("slow");
rig.dns.set("fast", &["192.168.23.9"]);
let began = tokio::time::Instant::now();
rig.pass(&perms, RESOLVE_EVERY).await.expect("renders");
assert!(
began.elapsed() <= RESOLVE_TIMEOUT,
"bounded by one timeout: {:?}",
began.elapsed()
);
assert_eq!(
rig.text(),
"atlas 192.168.23.2 2222\natlas 192.168.23.9 22\n",
"the other host's change landed and the hung host kept its answer"
);
assert_eq!(rig.answers.warned.len(), 1, "one warning for the timeout");
// A host that has never answered gets nothing, and the rest still land.
let mut fresh = Rig::new();
fresh.dns.set("fast", &["192.168.23.1"]);
fresh.dns.hang("slow");
fresh.pass(&perms, T0).await.expect("renders");
assert_eq!(fresh.text(), "atlas 192.168.23.1 22\n");
}
/// Several hung hosts cost one timeout together, not one each.
#[tokio::test(start_paused = true)]
async fn hung_lookups_run_together() {
let mut rig = Rig::new();
let hosts: Vec<String> = (0..6).map(|n| format!("h{n}")).collect();
for h in &hosts {
rig.dns.hang(h);
}
let conns: Vec<String> = hosts
.iter()
.enumerate()
.map(|(n, h)| format!(r#"{{"name":"c{n}","user":"root","host":"{h}","port":22}}"#))
.collect();
let perms = permissions::Stored::from([(
"atlas".to_owned(),
record(&format!(
r#"{{"ssh":true,"ssh_connections":[{}]}}"#,
conns.join(",")
)),
)]);
let began = tokio::time::Instant::now();
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(began.elapsed(), RESOLVE_TIMEOUT);
assert_eq!(rig.dns.calls(), 6);
}
#[test]
fn a_subnet_is_read_from_the_bridge_address_and_prefix() {
let net = Subnet::parse("10.42.0.1/24").expect("parses");
assert!(net.contains("10.42.0.200".parse().unwrap()));
assert!(net.contains("10.42.0.1".parse().unwrap()));
assert!(!net.contains("10.42.1.1".parse().unwrap()));
let wide = Subnet::parse("10.42.5.9/15").expect("parses");
assert!(wide.contains("10.43.255.1".parse().unwrap()));
assert!(!wide.contains("10.44.0.1".parse().unwrap()));
assert!(
Subnet::parse("0.0.0.0/0")
.expect("parses")
.contains("8.8.8.8".parse().unwrap())
);
for bad in ["10.42.0.1", "10.42.0.1/33", "nope/24", "10.42.0.1/x", ""] {
assert_eq!(Subnet::parse(bad), None, "{bad:?}");
}
}
#[tokio::test]
async fn an_address_inside_the_bridge_subnet_is_dropped_and_warned_once() {
let mut rig = Rig::new();
rig.bridge = Subnet::parse("10.42.0.1/24");
rig.dns.set("muede-pc2", &["10.42.0.7", "192.168.23.232"]);
let perms = permissions::Stored::from([("atlas".to_owned(), pc2("true"))]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "atlas 192.168.23.232 2222\n");
assert!(rig.answers.warned_bridge.contains("muede-pc2"));
// Control: without the subnet the same answer allows both.
let mut open = Rig::new();
open.dns.set("muede-pc2", &["10.42.0.7", "192.168.23.232"]);
open.pass(&perms, T0).await.expect("renders");
assert_eq!(
open.text(),
"atlas 10.42.0.7 2222\natlas 192.168.23.232 2222\n"
);
assert!(open.answers.warned_bridge.is_empty());
// The warning is armed again once the host leaves the subnet.
rig.dns.set("muede-pc2", &["192.168.23.232"]);
rig.pass(&perms, RESOLVE_EVERY).await.expect("renders");
assert!(rig.answers.warned_bridge.is_empty());
}
#[tokio::test]
async fn an_ip_literal_inside_the_bridge_subnet_allows_nothing() {
let mut rig = Rig::new();
rig.bridge = Subnet::parse("10.42.0.1/24");
let perms = permissions::Stored::from([(
"atlas".to_owned(),
record(
r#"{"ssh":true,"ssh_connections":[{"name":"x","user":"root","host":"10.42.0.9","port":22}]}"#,
),
)]);
rig.pass(&perms, T0).await.expect("renders");
assert_eq!(rig.text(), "");
}
}

View file

@ -1,13 +1,15 @@
//! Background tasks and periodic sweeps: crash/login watcher, boot-time
//! auto-update reconcile, the periodic declared ⇒ running reconcile tick, the agent-sockets.json writer loop, the MCP socket listener
//! reconcile loop, knowledge-repo sync, the swarm wanted-state pull, and the
//! swarm resource-cap and permissions reads, and the squid users file.
//! swarm resource-cap and permissions reads, the squid users file, and the
//! egress allowance for agents' SSH connections.
//! Each submodule is re-exported
//! at the crate root, so `crate::crash_watch::…` etc. keep working unchanged.
pub mod agent_sockets;
pub mod auto_update;
pub mod crash_watch;
pub mod egress_connections;
pub mod knowledge;
pub mod mcp_sockets;
pub mod permissions_import;

View file

@ -99,6 +99,12 @@ const LOCK: &str = "/run/lock/hive-egress-pin.lock";
/// `HIVE_EGRESS_UNRESTRICTED` names. Unset or unreadable lists no agent.
const UNRESTRICTED_ENV: &str = "HIVE_EGRESS_UNRESTRICTED";
/// The file of the agents' allowed connections, one `<agent> <ipv4> <port>` per
/// line, that `HIVE_EGRESS_CONNECTIONS` names. Unset leaves the `agent_allow` set
/// alone: a ruleset that does not declare it still loads. Set but unreadable
/// lists no connection.
const CONNECTIONS_ENV: &str = "HIVE_EGRESS_CONNECTIONS";
/// The prefix of an agent container's machine name.
const AGENT_PREFIX: &str = "h-";
@ -315,9 +321,66 @@ pub(crate) fn direct_macs(pins: &[Pin], unrestricted: &[String]) -> Vec<Mac> {
macs
}
/// One connection an agent may open: its name, the address and the TCP port.
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Connection {
pub(crate) agent: String,
pub(crate) ip: std::net::Ipv4Addr,
pub(crate) port: u16,
}
/// The connections in `text`, one `<agent> <ipv4> <port>` per line. A line of
/// another shape, a port 0 or a name that is no identifier is skipped.
pub(crate) fn parse_connections(text: &str) -> Vec<Connection> {
text.lines()
.filter_map(|line| {
let mut cols = line.split_whitespace();
let (agent, ip, port) = (cols.next()?, cols.next()?, cols.next()?);
if cols.next().is_some() {
return None;
}
hive_types::Ident::parse(agent).ok()?;
Some(Connection {
agent: agent.to_owned(),
ip: ip.parse().ok()?,
port: port.parse().ok().filter(|p| *p != 0)?,
})
})
.collect()
}
/// The `agent_allow` elements of `connections`: for each, one per MAC bound to
/// a port of the agent's container (`h-<agent>`). An agent with no binding has
/// none, so nothing is allowed for it until its port is bound.
pub(crate) fn connection_elements(pins: &[Pin], connections: &[Connection]) -> String {
let mut rows: Vec<(Mac, std::net::Ipv4Addr, u16)> = Vec::new();
for c in connections {
for p in pins {
if p.machine.strip_prefix(AGENT_PREFIX) == Some(c.agent.as_str()) {
rows.push((p.mac, c.ip, c.port));
}
}
}
rows.sort_by_key(|(m, ip, port)| (m.0, *ip, *port));
rows.dedup();
let mut script = String::new();
for (mac, ip, port) in rows {
let _ = writeln!(
script,
"add element {INET_TABLE} agent_allow {{ {mac} . {ip} . {port} }}"
);
}
script
}
/// The nft input that adds the bindings `pins` to the two sets of the bridge
/// table and the [`direct_macs`] to the exemption set of the inet table.
pub(crate) fn pin_elements(pins: &[Pin], unrestricted: &[String]) -> String {
/// table, the [`direct_macs`] to the exemption set of the inet table and, when
/// `connections` is given, its [`connection_elements`].
pub(crate) fn pin_elements(
pins: &[Pin],
unrestricted: &[String],
connections: Option<&[Connection]>,
) -> String {
let mut script = String::new();
for Pin { port, mac, .. } in pins {
let _ = writeln!(
@ -330,17 +393,30 @@ pub(crate) fn pin_elements(pins: &[Pin], unrestricted: &[String]) -> String {
for mac in direct_macs(pins, unrestricted) {
let _ = writeln!(script, "add element {INET_TABLE} direct_mac {{ {mac} }}");
}
if let Some(connections) = connections {
script.push_str(&connection_elements(pins, connections));
}
script
}
/// The nft input that makes the bindings and the exemptions exactly those of
/// `pins` and `unrestricted`, in one transaction: a reader never sees a set
/// empty in between.
pub(crate) fn pin_script(pins: &[Pin], unrestricted: &[String]) -> String {
/// The nft input that makes the bindings, the exemptions and, when
/// `connections` is given, the allowed connections exactly those of `pins`,
/// `unrestricted` and `connections`, in one transaction: a reader never sees a
/// set empty in between.
pub(crate) fn pin_script(
pins: &[Pin],
unrestricted: &[String],
connections: Option<&[Connection]>,
) -> String {
let flush_connections = if connections.is_some() {
format!("flush set {INET_TABLE} agent_allow\n")
} else {
String::new()
};
format!(
"flush set {TABLE} port_mac\nflush set {TABLE} port_chaddr\n\
flush set {INET_TABLE} direct_mac\n{}",
pin_elements(pins, unrestricted)
flush set {INET_TABLE} direct_mac\n{flush_connections}{}",
pin_elements(pins, unrestricted, connections)
)
}
@ -360,6 +436,15 @@ fn unrestricted_names() -> Vec<String> {
.collect()
}
/// The connections in the file `HIVE_EGRESS_CONNECTIONS` names, or `None` when
/// it names none. A file that cannot be read lists no connection.
fn connection_list() -> Option<Vec<Connection>> {
let path = std::env::var_os(CONNECTIONS_ENV).filter(|p| !p.is_empty())?;
Some(parse_connections(
&std::fs::read_to_string(path).unwrap_or_default(),
))
}
/// The agent whose container holds the IPv4 address `ip`, as dnsmasq's lease
/// file and the bindings of the ports say: the lease names a MAC, the MAC is
/// bound to one port, and that port's machine is `h-<agent>`. An address in the
@ -900,7 +985,11 @@ fn pin(args: &[String]) -> Result<()> {
let survey = {
let _lock = lock(deadline)?;
let survey = survey(&bridge)?;
nft(&pin_script(&survey.resolution.pins, &unrestricted_names()))?;
nft(&pin_script(
&survey.resolution.pins,
&unrestricted_names(),
connection_list().as_deref(),
))?;
commit(&survey)?;
survey
};
@ -964,7 +1053,11 @@ fn load(args: &[String]) -> Result<()> {
let _lock = lock(Instant::now() + WAIT)?;
let found = survey(&bridge);
let pins = match &found {
Ok(found) => pin_elements(&found.resolution.pins, &unrestricted_names()),
Ok(found) => pin_elements(
&found.resolution.pins,
&unrestricted_names(),
connection_list().as_deref(),
),
Err(e) => {
eprintln!("{LOAD}: binding the ports failed, loading without bindings: {e:#}");
String::new()
@ -1241,7 +1334,7 @@ mod tests {
#[test]
fn the_ruleset_loads_with_the_bindings_and_leases_in_one_script() {
let calls = std::cell::RefCell::new(Vec::new());
let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]);
let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[], None);
let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap();
load_ruleset("table bridge hive_egress { }", &pins, &leases, |s| {
calls.borrow_mut().push(s.to_string());
@ -1260,7 +1353,7 @@ mod tests {
#[test]
fn a_lease_load_that_fails_is_a_warning_and_the_bindings_still_load() {
let calls = std::cell::RefCell::new(Vec::new());
let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[]);
let pins = pin_elements(&[pin("vb-a", 10, 10, "a")], &[], None);
let leases = lease_script("add", "02:00:00:00:00:0a", "10.42.0.10", 90).unwrap();
let result = load_ruleset("RULESET", &pins, &leases, |s| {
calls.borrow_mut().push(s.to_string());
@ -1284,7 +1377,7 @@ mod tests {
#[test]
fn pin_script_replaces_all_three_sets_in_one_script() {
let script = pin_script(&[pin("vb-h-a", 10, 10, "h-a")], &[]);
let script = pin_script(&[pin("vb-h-a", 10, 10, "h-a")], &[], None);
assert_eq!(
script,
"flush set bridge hive_egress port_mac\n\
@ -1293,7 +1386,7 @@ mod tests {
add element bridge hive_egress port_mac { \"vb-h-a\" . 02:00:00:00:00:0a }\n\
add element bridge hive_egress port_chaddr { \"vb-h-a\" . 0x02000000000a }\n"
);
assert!(!pin_script(&[], &[]).contains("add element"));
assert!(!pin_script(&[], &[], None).contains("add element"));
}
/// Service containers and the agents the file lists are exempt; an agent the
@ -1310,13 +1403,102 @@ mod tests {
assert_eq!(direct_macs(&pins, &listed), vec![mac(2), mac(3)]);
// Control: with nothing listed, only the service container stays.
assert_eq!(direct_macs(&pins, &[]), vec![mac(3)]);
let script = pin_elements(&pins, &listed);
let script = pin_elements(&pins, &listed, None);
assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:02 }"));
assert!(script.contains("add element inet hive_egress direct_mac { 02:00:00:00:00:03 }"));
assert!(!script.contains("direct_mac { 02:00:00:00:00:01 }"));
assert!(!script.contains("direct_mac { 02:00:00:00:00:04 }"));
}
fn conn(agent: &str, ip: &str, port: u16) -> Connection {
Connection {
agent: agent.into(),
ip: ip.parse().unwrap(),
port,
}
}
/// An agent's connection is allowed from the MAC of that agent's port only,
/// not from a container whose name merely starts the same or from a service
/// container.
#[test]
fn a_connection_is_allowed_from_its_agents_mac_only() {
let pins = [
pin("vb-h-atlas", 10, 1, "h-atlas"),
pin("vb-h-atlasx", 11, 2, "h-atlasx"),
pin("vb-matrix", 12, 3, "atlas"),
];
let script = connection_elements(&pins, &[conn("atlas", "192.168.23.232", 2222)]);
assert_eq!(
script,
"add element inet hive_egress agent_allow \
{ 02:00:00:00:00:01 . 192.168.23.232 . 2222 }\n"
);
}
#[test]
fn a_connection_of_an_unbound_agent_adds_nothing_and_duplicates_collapse() {
let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")];
assert_eq!(
connection_elements(&pins, &[conn("uma", "10.0.0.1", 22)]),
""
);
let twice = [conn("atlas", "10.0.0.1", 22), conn("atlas", "10.0.0.1", 22)];
assert_eq!(connection_elements(&pins, &twice).lines().count(), 1);
assert_eq!(connection_elements(&[], &twice), "");
}
#[test]
fn the_connections_file_is_parsed_line_by_line_and_bad_lines_are_skipped() {
let text = "atlas 192.168.23.232 2222\n\
argus 10.0.0.1 22\n\
\n\
atlas not-an-ip 22\n\
atlas 10.0.0.2 0\n\
atlas 10.0.0.2 70000\n\
bad;name 10.0.0.3 22\n\
atlas 10.0.0.4 22 extra\n\
atlas 10.0.0.5\n";
assert_eq!(
parse_connections(text),
[
conn("atlas", "192.168.23.232", 2222),
conn("argus", "10.0.0.1", 22)
]
);
}
/// With a connections file the script flushes `agent_allow` in the same
/// transaction that refills it; without one it does not name the set, so a
/// ruleset that never declared it still loads.
#[test]
fn the_script_flushes_and_fills_agent_allow_only_when_a_file_is_named() {
let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")];
let with = pin_script(&pins, &[], Some(&[conn("atlas", "10.0.0.1", 22)]));
let flush = with
.find("flush set inet hive_egress agent_allow\n")
.expect("flushed");
let add = with
.find("add element inet hive_egress agent_allow { 02:00:00:00:00:01 . 10.0.0.1 . 22 }")
.expect("added");
assert!(flush < add, "the flush comes first");
// Control: a named file with no connection still flushes the old ones.
let empty = pin_script(&pins, &[], Some(&[]));
assert!(empty.contains("flush set inet hive_egress agent_allow\n"));
assert!(!empty.contains("add element inet hive_egress agent_allow"));
let none = pin_script(&pins, &[], None);
assert!(!none.contains("agent_allow"), "{none}");
assert!(none.contains("flush set inet hive_egress direct_mac"));
}
#[test]
fn the_load_script_fills_agent_allow_without_flushing_it() {
let pins = [pin("vb-h-atlas", 10, 1, "h-atlas")];
let script = pin_elements(&pins, &[], Some(&[conn("atlas", "10.0.0.1", 22)]));
assert!(script.contains("agent_allow { 02:00:00:00:00:01 . 10.0.0.1 . 22 }"));
assert!(!script.contains("flush"));
}
const NOW: u64 = 1_000;
fn lease_line(expiry: u64, last: u8, ip: &str) -> String {

View file

@ -298,6 +298,14 @@ in
HIVE_SQUID_PROXY = config.services.hyperhive.deploy.hive-controller.squid.listen;
HIVE_EGRESS_NO_PROXY = lib.concatStringsSep "," egressDirectHosts;
}
//
lib.optionalAttrs (config.services.hyperhive.deploy.hive-controller.egress.connectionsFile != null)
{
# Where `workers::egress_connections` renders the SSH connections each agent
# may open, for the egress forward chain.
HIVE_EGRESS_CONNECTIONS_FILE =
config.services.hyperhive.deploy.hive-controller.egress.connectionsFile;
}
// lib.optionalAttrs (haveBaoClientIdentity && baoDeploy.serverCaFile != null) {
# Absent means the system trust store — right for a deployment with a real
# CA, wrong for the self-signed one ../glue-bao-tls.nix mints, which is why

View file

@ -74,6 +74,9 @@ let
}
// lib.optionalAttrs (egressCfg.intercept.unrestrictedFile != null) {
HIVE_EGRESS_UNRESTRICTED = toString egressCfg.intercept.unrestrictedFile;
}
// lib.optionalAttrs (egressCfg.connectionsFile != null) {
HIVE_EGRESS_CONNECTIONS = toString egressCfg.connectionsFile;
};
leaseScript = pkgs.writeShellScript "hive-egress-lease" ''
export PATH=${lib.makeBinPath [ pkgs.nftables ]}
@ -186,6 +189,7 @@ let
chain forward {
type filter hook forward priority filter - 10; policy accept;
${fromBridge} ether saddr @direct_mac accept
${fromBridge} meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept
${fromBridge} counter
${lib.concatStringsSep "\n " (
logRules "v4" "ip" ++ logRules "v6" "ip6" ++ unmeteredRules ++ finalRules
@ -309,6 +313,7 @@ let
${lib.optionalString refuseEch ''iifname "${br}" ip saddr ${relayPool} counter drop''}
}
set direct_mac { type ether_addr; }
set agent_allow { type ether_addr . ipv4_addr . inet_service; }
${lib.optionalString egressCfg.intercept.enable interceptChain}
${lib.optionalString (egressCfg.forward != "accept") forwardChain}
}
@ -342,7 +347,9 @@ in
What the `inet hive_egress` table does with traffic the host forwards
from the agent bridge to anywhere but the bridge. In `log` and
`reject` the traffic of the MACs in the `direct_mac` set (the unrestricted
agents and the service containers) is accepted and never logged.
agents and the service containers) is accepted and never logged, and so
is TCP from an agent to an address and port in the `agent_allow` set
(`connectionsFile`).
- `accept`: nothing; the table has no forward chain.
- `log`: each new flow is logged with the prefix `hive-egress-would-drop `,
@ -359,6 +366,22 @@ in
'';
};
options.services.hyperhive.deploy.hive-controller.egress.connectionsFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The file hive-c0re renders with the SSH connections each agent may open,
one `<agent> <ipv4> <port>` per line: the addresses behind the `host` of
each connection in the permissions record of an agent whose `ssh` switch
is on. `hive-priv egress-pin` turns each line into elements of the
`agent_allow` set of the `inet hive_egress` table, keyed by the MAC of
the agent's port, and the forward chain accepts TCP to an address and
port in it before it logs or refuses. The file is read when a port is
bound, on each pass of the 30 second timer and when the file changes.
`null` leaves the set alone, empty: no agent has an allowance.
'';
};
options.services.hyperhive.deploy.hive-controller.egress.antiSpoof = lib.mkOption {
type = lib.types.enum [
"drop"
@ -664,6 +687,17 @@ in
};
};
# The same pass refills `agent_allow`, so a connection an operator declares
# is allowed at once.
systemd.paths.hive-egress-connections = lib.mkIf (egressCfg.connectionsFile != null) {
description = "Watch the agents' allowed connections list for the egress forward chain";
wantedBy = [ "paths.target" ];
pathConfig = {
PathChanged = toString egressCfg.connectionsFile;
Unit = "hive-egress-pin.service";
};
};
# nspawn names the host end of a container's veth `vb-<machine>` when the
# container joins a bridge.
services.udev.extraRules = ''

View file

@ -43,6 +43,7 @@ let
proxyUrl = "http://${netCfg.bridgeIp}:${toString port}";
exporterListen = "127.0.0.1:9301";
interceptCfg = hyperhiveCfg.deploy.hive-controller.egress.intercept;
egressCfg = hyperhiveCfg.deploy.hive-controller.egress;
# The path every platform container's unit runs hive-priv by (hive-priv.nix
# points it at the current build). Squid runs `egress-whois` through it.
hivePriv = (import ./lib/container-rootfs-check.nix { inherit config; }).hivePriv;
@ -555,6 +556,11 @@ in
enable = lib.mkDefault true;
inherit (cfg) unrestrictedFile agentAllowFile;
};
# The file `hive-c0re` renders with the SSH connections each agent may open,
# which the egress forward chain reads (`workers::egress_connections`). It
# sits in the directory hive-c0re may write.
services.hyperhive.deploy.hive-controller.egress.connectionsFile =
lib.mkDefault "${usersDir}/ssh-connections";
services.hyperhive.network.enable = lib.mkDefault true;
services.hyperhive.otel.scrapeTargets.squid = exporterListen;
@ -591,7 +597,10 @@ in
"C ${cfg.allowedFile} 0640 hive-core squid - ${seedFile}"
"f ${cfg.agentAllowFile} 0640 hive-core squid - -"
"f ${cfg.denyFile} 0640 hive-core squid - -"
];
]
++ lib.optional (
egressCfg.connectionsFile != null
) "f ${egressCfg.connectionsFile} 0640 hive-core squid - -";
# Squid reads the unrestricted list, the allow-list and the per-agent include
# when it parses its configuration, not when they change. hive-c0re writes

View file

@ -1645,6 +1645,57 @@ let
&& !(lib.any (l: lib.hasSuffix " drop" l || lib.hasSuffix " reject" l) rules)
&& lib.hasInfix "chain input" text;
}
{
# An agent's declared SSH connections are accepted before the chain
# counts, logs or refuses, in `log` and in `reject`. The set takes MAC,
# address and port. The host without a controller has the set and no
# chain (the presence control) and no wiring of the connections file.
# hive-c0re, the units that fill the sets and the path unit that runs a
# pass are told the same file.
name = "the egress forward chain accepts the agents' declared ssh connections before it logs or refuses";
ok =
let
rulesetOf = m: m.services.hyperhive.network.egress.ruleset;
inetOf = m: lib.head (lib.splitString "table bridge hive_egress {" (rulesetOf m));
rulesOf =
m:
lib.filter (l: l != "" && l != "}") (
map lib.trim (lib.splitString "\n" (lib.last (lib.splitString "chain forward" (inetOf m))))
);
idxIn = rules: pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
allow = ''iifname "hive-br0" oifname != "hive-br0" meta nfproto ipv4 meta l4proto tcp ether saddr . ip daddr . tcp dport @agent_allow accept'';
set = "set agent_allow { type ether_addr . ipv4_addr . inet_service; }";
inOrder =
m:
let
rules = rulesOf m;
idx = idxIn rules;
in
lib.elem allow rules
&& lib.length (lib.filter (r: r == allow) rules) == 1
&& idx "ether saddr @direct_mac accept" < idx "@agent_allow accept"
&& idx "@agent_allow accept" < idx "counter"
&& idx "@agent_allow accept" < idx "update @log_v4"
&& lib.hasInfix set (inetOf m);
file = "/var/lib/hive-squid/ssh-connections";
envOf = m: name: m.systemd.services.${name}.environment;
in
inOrder bare
&& inOrder egressForwardReject
&& lib.hasInfix set (inetOf egressForwardAccept)
&& !(lib.hasInfix "@agent_allow accept" (rulesetOf egressForwardAccept))
&& bare.services.hyperhive.deploy.hive-controller.egress.connectionsFile == file
&& bare.systemd.services.hive-c0re.environment.HIVE_EGRESS_CONNECTIONS_FILE == file
&& lib.all (name: ((envOf bare name).HIVE_EGRESS_CONNECTIONS or null) == file) [
"hive-egress"
"hive-egress-pin@"
"hive-egress-pin"
]
&& bare.systemd.paths.hive-egress-connections.pathConfig.PathChanged == file
&& bare.systemd.paths.hive-egress-connections.pathConfig.Unit == "hive-egress-pin.service"
&& !((envOf egressForwardAccept "hive-egress") ? HIVE_EGRESS_CONNECTIONS)
&& !(egressForwardAccept.systemd.paths ? hive-egress-connections);
}
{
# The bridge table drops what a port sends from an address that is not
# its own, and logs it. The ruleset is read as text: each `spoof_<reason>`
@ -1807,7 +1858,7 @@ let
chain = lib.last (lib.splitString "chain forward" text);
rules = lib.filter (l: l != "" && l != "}") (map lib.trim (lib.splitString "\n" chain));
idx = pat: lib.lists.findFirstIndex (lib.hasInfix pat) 1000 rules;
tcp = lib.filter (lib.hasInfix " meta l4proto tcp ") rules;
tcp = lib.filter (l: lib.hasInfix " meta l4proto tcp " l && !(lib.hasInfix "@agent_allow" l)) rules;
other = lib.filter (lib.hasInfix " meta l4proto != tcp") rules;
logged = lib.filter (lib.hasInfix "log prefix") rules;
bareText = bare.services.hyperhive.network.egress.ruleset;
@ -1820,7 +1871,7 @@ let
&& !(lib.hasInfix "wg-hive" text)
&& idx "ether saddr @direct_mac accept" < idx "counter"
&& idx "counter" < idx "reject"
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 1
&& lib.length (lib.filter (lib.hasSuffix " accept") rules) == 2
&& lib.length tcp == 4
&& lib.length other == 4
&& lib.all (lib.hasSuffix " reject with tcp reset") tcp