diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 25be4475b..294a30a80 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -415,6 +415,17 @@ Destroying the container removes the drop-in and then its root; the build stays until the next collection. A drop-in written by an older hive-priv has no `ExecStartPre=` and no root, and the container starts as before. +**Hyperhive's platform containers: warn only.** The forge, the matrix server and +the swarm services (15 containers, declared by their own modules) run the same +check as `hive-priv check-rootfs --warn-only `, from `ExecStartPre=` on +their own `container@` unit. It evaluates every rule, logs each violation +as a warning, and exits 0, so the container starts either way. The line reads +`rootfs check (warn-only): : ` with journald's warning priority. +Strict refusal needs the live roots of all 15 to pass first, and the warnings +show which ones would not. A container the operator runs that hyperhive doesn't +declare gets no check at all: the modules name their own containers, and +nothing enumerates the host's `containers`. + **Disk use.** The copy shares blocks with the old root where it can: hive-priv clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and copies the data of a file the filesystem refuses to clone. Before it creates diff --git a/hive-priv/src/prestart.rs b/hive-priv/src/prestart.rs index 5faccc751..b0a0683e2 100644 --- a/hive-priv/src/prestart.rs +++ b/hive-priv/src/prestart.rs @@ -3,12 +3,17 @@ //! nixpkgs' `container@.service` start script runs as host root against the //! container's own root, which is persistent and writable by the container. //! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a -//! link the container planted at any of the names the script touches. The -//! per-container drop-in ([`crate::container_config`]) puts +//! link the container planted at any of the names the script touches. An agent +//! container's drop-in ([`crate::container_config`]) puts //! `hive-priv check-rootfs ` ahead of that script as an //! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped //! while this runs, so nothing changes under the walk. //! +//! Hyperhive's own declarative containers (the forge, the matrix server, the +//! swarm services) run it as `check-rootfs --warn-only ` from their +//! own unit (`nix/host-modules/lib/container-rootfs-check.nix`): each violation +//! is logged at warning level and the start goes on. +//! //! Every component is looked at with `symlink_metadata`: a link is reported, //! never followed. Nothing is deleted or repaired, so whatever is there stays //! for the operator. @@ -27,11 +32,15 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use crate::container_config::{container_rootfs, validate_container_system_name}; -use crate::shared::partial_name; +use crate::shared::{partial_name, validate_name_chars}; /// The sub-command the drop-in's `ExecStartPre=` names. pub(crate) const SUBCOMMAND: &str = "check-rootfs"; +/// The flag that turns a violation into a warning: the check runs in full and +/// exits 0. +const WARN_ONLY: &str = "--warn-only"; + /// The per-container nix profile directories on the host. const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container"; @@ -135,33 +144,84 @@ fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> { /// Handle `hive-priv ...` when the first argument names one. /// `None` means it is not a sub-command and the daemon starts. /// -/// Prints one line to stderr and returns exit code 1 on a violation, so the -/// reason lands in the journal next to the failed unit. +/// `check-rootfs ` is strict: it prints one line to stderr and +/// returns exit code 1 on a violation, so the reason lands in the journal next +/// to the failed unit. `check-rootfs --warn-only ` prints every +/// violation as a warning and returns 0. pub(crate) fn run(args: &[String]) -> Option { if args.first().map(String::as_str) != Some(SUBCOMMAND) { return None; } - let result = match args { - [_, container] => check_container(container), - _ => Err(anyhow::anyhow!("usage: hive-priv {SUBCOMMAND} ")), - }; - Some(match result { - Ok(()) => 0, - Err(e) => { - eprintln!("hive-priv {SUBCOMMAND}: refusing to start: {e:#}"); - 1 + let (container, warn_only) = match args { + [_, container] => (container, false), + [_, flag, container] if flag == WARN_ONLY => (container, true), + _ => { + eprintln!("usage: hive-priv {SUBCOMMAND} [{WARN_ONLY}] "); + return Some(1); } - }) + }; + let found = found_in_container(container, warn_only); + let (code, lines) = report(container, &found, warn_only); + for line in lines { + eprintln!("{line}"); + } + Some(code) } -/// [`check`] on the host paths of `container`. -fn check_container(container: &str) -> Result<()> { - validate_container_system_name(container)?; - check( - &container_rootfs(container), - &Path::new(PROFILES_ROOT).join(container).join("system"), - Path::new(STORE), - ) +/// What a start would be refused for, on the host paths of `container`. +/// +/// Strict mode stops at the first violation, as the start would. Warn-only +/// lists them all, and counts a name it can't use as one more: it never fails +/// the start over its own input. +fn found_in_container(container: &str, warn_only: bool) -> Vec { + let paths = container_paths(container, warn_only); + match (paths, warn_only) { + (Ok((root, system)), true) => violations(&root, &system, Path::new(STORE)), + (Ok((root, system)), false) => check(&root, &system, Path::new(STORE)) + .err() + .map(|e| format!("{e:#}")) + .into_iter() + .collect(), + (Err(e), _) => vec![format!("{e:#}")], + } +} + +/// The container's root and its profile's `system` link on the host. +/// +/// An agent container is named `h-`. A warn-only check also takes the +/// platform containers' names, which are lowercase ascii, digits and hyphens. +/// Either way the name holds no `/` and is never `.` or `..`. +fn container_paths(container: &str, warn_only: bool) -> Result<(PathBuf, PathBuf)> { + if warn_only { + validate_name_chars(container)?; + } else { + validate_container_system_name(container)?; + } + Ok(( + container_rootfs(container), + Path::new(PROFILES_ROOT).join(container).join("system"), + )) +} + +/// The exit code and the stderr lines for the violations `found`. +/// +/// Warn-only lines carry the `<4>` prefix, which journald reads as the +/// warning priority, so a log query can select them by severity. +fn report(container: &str, found: &[String], warn_only: bool) -> (i32, Vec) { + if warn_only { + let lines = found + .iter() + .map(|v| format!("<4>rootfs check (warn-only): {container}: {v}")) + .collect(); + return (0, lines); + } + match found.first() { + Some(v) => ( + 1, + vec![format!("hive-priv {SUBCOMMAND}: refusing to start: {v}")], + ), + None => (0, Vec::new()), + } } /// Check the paths the start script touches as root, returning an error that @@ -173,14 +233,36 @@ fn check_container(container: &str) -> Result<()> { /// A path that does not exist passes: a first start has nothing yet, and the /// script creates it. fn check(root: &Path, system: &Path, store: &Path) -> Result<()> { + match violations(root, system, store).into_iter().next() { + None => Ok(()), + Some(first) => bail!("{first}"), + } +} + +/// Every path under the rules of [`check`] that holds something it must not, +/// in the order [`check`] looks at them, each message once. +fn violations(root: &Path, system: &Path, store: &Path) -> Vec { + let mut found = Vec::new(); for dir in DIRS { - check_dir(root, dir)?; + note(&mut found, check_dir(root, dir)); } for file in REGULAR_ONLY { - check_file(root, file, false)?; + note(&mut found, check_file(root, file, false)); + } + note(&mut found, check_file(root, OS_RELEASE, true)); + note(&mut found, check_system(system, store)); + found +} + +/// Add the message of an error to `found`, unless it is already there: a link +/// at `usr` is the finding for `usr/bin` too. +fn note(found: &mut Vec, result: Result<()>) { + if let Err(e) = result { + let message = format!("{e:#}"); + if !found.contains(&message) { + found.push(message); + } } - check_file(root, OS_RELEASE, true)?; - check_system(system, store) } /// `symlink_metadata`, with `None` for a path that does not exist. The one @@ -285,7 +367,7 @@ mod tests { use std::os::unix::fs::symlink; use std::path::PathBuf; - use super::{check, pin_binary_in, run, unpin_binary_in}; + use super::{check, container_paths, pin_binary_in, report, run, unpin_binary_in, violations}; use crate::shared::test_support::{entries, scratch}; /// A scratch container: root, the profile link's path and a store with one @@ -608,16 +690,133 @@ mod tests { assert!(msg.contains("left alone"), "{msg}"); assert!(roots.join("h-iris/keep").exists()); } + #[test] fn only_the_check_rootfs_sub_command_is_taken() { + let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::>(); assert_eq!(run(&[]), None); - assert_eq!(run(&["serve".to_string()]), None); - // Wrong arity and a name that is not an agent container are refused, - // whatever is on disk. - assert_eq!(run(&["check-rootfs".to_string()]), Some(1)); + assert_eq!(run(&args(&["serve"])), None); + // Wrong arity, an unknown flag and a name that is not an agent + // container are refused, whatever is on disk. + assert_eq!(run(&args(&["check-rootfs"])), Some(1)); + assert_eq!(run(&args(&["check-rootfs", "not-hive"])), Some(1)); + assert_eq!(run(&args(&["check-rootfs", "--strict", "h-iris"])), Some(1)); + assert_eq!(run(&args(&["check-rootfs", "--warn-only"])), Some(1)); + } + + #[test] + fn a_warn_only_run_exits_zero_even_for_a_name_it_cannot_use() { + let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::>(); assert_eq!( - run(&["check-rootfs".to_string(), "not-hive".to_string()]), - Some(1) + run(&args(&["check-rootfs", "--warn-only", "no-such-container"])), + Some(0) + ); + assert_eq!( + run(&args(&["check-rootfs", "--warn-only", "../etc"])), + Some(0) ); } + + #[test] + fn strict_takes_agent_names_and_warn_only_takes_plain_ones() { + assert!(container_paths("h-iris", false).is_ok()); + assert!(container_paths("hive-forge", false).is_err()); + for name in ["hive-forge", "swarm-review-classifier", "h-iris"] { + let (root, system) = container_paths(name, true).unwrap(); + assert_eq!( + root, + PathBuf::from(format!("/var/lib/nixos-containers/{name}")) + ); + assert_eq!( + system, + PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system")) + ); + } + for name in [ + "", + ".", + "..", + "../h-iris", + "a/b", + "/etc", + "H-iris", + "a.b", + "a b", + ] { + assert!( + container_paths(name, true).is_err(), + "{name:?} was accepted" + ); + assert!( + container_paths(name, false).is_err(), + "{name:?} was accepted" + ); + } + } + + /// A clean root with a link at `root` and another at `etc/resolv.conf`. + fn two_violations() -> Fixture { + let f = Fixture::clean(); + std::fs::remove_dir_all(f.root.join("root")).unwrap(); + symlink(victim(), f.root.join("root")).unwrap(); + std::fs::remove_file(f.root.join("etc/resolv.conf")).unwrap(); + symlink("/etc/hosts", f.root.join("etc/resolv.conf")).unwrap(); + f + } + + fn found(f: &Fixture) -> Vec { + violations(&f.root, &f.system, &f.store) + } + + #[test] + fn warn_only_logs_every_violation_at_warning_level_and_exits_zero() { + let f = two_violations(); + let found = found(&f); + assert_eq!(found.len(), 2, "{found:?}"); + let (code, lines) = report("hive-forge", &found, true); + assert_eq!(code, 0); + assert_eq!(lines.len(), 2, "{lines:?}"); + for line in &lines { + assert!( + line.starts_with("<4>rootfs check (warn-only): hive-forge: "), + "{line}" + ); + } + assert!(lines[0].contains("root is a symlink"), "{lines:?}"); + assert!(lines[1].contains("resolv.conf is a symlink"), "{lines:?}"); + } + + #[test] + fn strict_refuses_with_the_first_violation_only() { + let f = two_violations(); + let found = found(&f); + let (code, lines) = report("h-iris", &found, false); + assert_eq!(code, 1); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!( + lines[0].starts_with("hive-priv check-rootfs: refusing to start: "), + "{lines:?}" + ); + assert!(lines[0].contains("root is a symlink"), "{lines:?}"); + // The rules are the same: the strict refusal is `check`'s first error. + assert_eq!(f.refused(), found[0]); + } + + #[test] + fn a_clean_root_prints_nothing_in_either_mode() { + let f = Fixture::clean(); + assert!(found(&f).is_empty()); + assert_eq!(report("hive-forge", &[], true), (0, Vec::new())); + assert_eq!(report("h-iris", &[], false), (0, Vec::new())); + } + + #[test] + fn one_link_is_reported_once_though_two_paths_run_through_it() { + let f = Fixture::clean(); + std::fs::remove_dir_all(f.root.join("usr")).unwrap(); + symlink(victim(), f.root.join("usr")).unwrap(); + let found = found(&f); + assert_eq!(found.len(), 1, "{found:?}"); + assert!(found[0].contains("usr is a symlink"), "{found:?}"); + } } diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 647ea9954..565bbbc37 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -78,6 +78,8 @@ let deployCfg.bao.forgePublisherClientCertFile != null && deployCfg.bao.forgePublisherClientKeyFile != null; + rootfsCheck = import ../lib/container-rootfs-check.nix { inherit config; }; + caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # The container's journal forwarder pushes to its own receiver on the swarm @@ -471,6 +473,7 @@ in # ../swarm-builder.nix's does. systemd.services."container@hive-forge" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly "hive-forge") { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index d691cbbac..c77a847f2 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -13,6 +13,8 @@ let # the client is pointed at the server name instead. hyperhiveDomain = config.services.hyperhive.domain; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # Same runtime→build-time bridge hive-forge and swarm-ci-runner cross: # binds the hive trust bundle (which folds in the store's root) into the # container and orders the container after `hive-tls-ca.service`. The @@ -1096,6 +1098,7 @@ in systemd.services."container@hive-matrix" = lib.mkMerge [ { after = [ "dnsmasq.service" ]; } caTrust.containerOrdering + (rootfsCheck.warnOnly "hive-matrix") ]; }; } diff --git a/nix/host-modules/lib/container-rootfs-check.nix b/nix/host-modules/lib/container-rootfs-check.nix new file mode 100644 index 000000000..5922c80a7 --- /dev/null +++ b/nix/host-modules/lib/container-rootfs-check.nix @@ -0,0 +1,34 @@ +# The pre-start check of a container's root, for one of hyperhive's own +# declarative containers. +# +# nixpkgs' start script runs as host root against the container's own root, +# which is persistent and writable by the container, and its `mkdir`, `chmod`, +# `touch` and `cp` follow links the container planted there. `hive-priv +# check-rootfs` looks at the paths the script touches (rules in +# `hive-priv/src/prestart.rs`). With `--warn-only` it logs each violation at +# warning level and lets the start go on. The line carries systemd's `-` +# prefix, so a crash or a failed exec of the check cannot fail the start either. +# The agent containers run it strict +# from their per-agent drop-in (hive-priv's `WriteResourceLimits`). +# +# Pure function, not a NixOS module. A container's module calls it with its +# own name and folds the result into its unit: +# +# rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; +# # systemd.services."container@${machine}" = lib.mkMerge [ (rootfsCheck.warnOnly machine) … ]; +# +# The name list is the callers', never `config.containers`: that also holds +# containers the operator runs that hyperhive doesn't own, and those get +# nothing. nixpkgs renders each declarative container as its own +# `container@` unit (`nixos-containers.nix`, the `mapAttrsToList` that +# builds `container@${name}`), so a setting on that unit reaches that +# container only. +{ config }: +{ + # Attributes for `systemd.services."container@"`. + warnOnly = name: { + serviceConfig.ExecStartPre = [ + "-${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}" + ]; + }; +} diff --git a/nix/host-modules/swarm-attic.nix b/nix/host-modules/swarm-attic.nix index de203d2e1..9b6f0477a 100644 --- a/nix/host-modules/swarm-attic.nix +++ b/nix/host-modules/swarm-attic.nix @@ -114,6 +114,8 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The forwarder reaches the swarm collector and authelia by name over TLS; in # self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { @@ -226,6 +228,7 @@ in # CA ordering keeps the trust bundle's bind source there first. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 2e5911775..39783827f 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -183,6 +183,8 @@ let refreshConsumer = import ./lib/refresh-consumer.nix { }; storeSeededSecret = import ./lib/store-seeded-secret.nix { }; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + caTrust = import ./lib/hive-ca-trust.nix { inherit lib gatewayCfg; tlsCfg = deployCfg.hive-controller.tls; @@ -970,6 +972,7 @@ in # not be up. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly cfg.machine) (lib.mkIf publishes { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index dd5b1bab7..9e6df0ea0 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1689,6 +1689,8 @@ let forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}"; autheliaCfg = hyperhiveCfg.swarm.authelia; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # Outbound TLS for that collector: it dials the swarm collector's gateway # name and authelia's token endpoint, both leaves under the store's runtime # pki root, which cannot be baked into a derivation. Same helper, same two @@ -4091,6 +4093,7 @@ in # cannot start is a store that is down. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly cfg.machine) { after = [ "systemd-tmpfiles-resetup.service" ]; } ]; diff --git a/nix/host-modules/swarm-builder.nix b/nix/host-modules/swarm-builder.nix index 1fb12bd72..70234fca2 100644 --- a/nix/host-modules/swarm-builder.nix +++ b/nix/host-modules/swarm-builder.nix @@ -52,6 +52,8 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8892; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The daemon reaches authelia, the store and the queue by name over TLS, and # `git` reaches the forge; in self-signed mode their leaves chain to the hive # CA. @@ -165,6 +167,7 @@ in # the unit does not exist. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index 690b0348b..a67cd9c56 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -43,6 +43,8 @@ let # below compares the two, so both come from here. forgeUrl = "https://${forgeCfg.domain}"; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The forge's gateway serves a leaf from the store's `pki` mount in # self-signed mode. # The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store, @@ -329,6 +331,7 @@ in # minting unit in and waits for it, as ./swarm-builder.nix's does. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-claude-login.nix b/nix/host-modules/swarm-claude-login.nix index 1c75147f4..c7d3b7826 100644 --- a/nix/host-modules/swarm-claude-login.nix +++ b/nix/host-modules/swarm-claude-login.nix @@ -54,6 +54,8 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8893; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -146,6 +148,7 @@ in # missing, so where this host mints the leaf the container waits for it. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; } ]; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index a5bbb2f43..49853a4ee 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -24,6 +24,8 @@ let baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = deployCfg.bao; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + caTrust = import ./lib/hive-ca-trust.nix { inherit lib gatewayCfg; tlsCfg = deployCfg.hive-controller.tls; @@ -686,6 +688,7 @@ in # ./swarm-builder.nix's does. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly cfg.machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 45b857e16..845c838c5 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -33,6 +33,8 @@ let tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; gatewayCfg = config.services.hyperhive.gateway; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # The responder introspects authelia over https BY NAME. Its HTTP client is # reqwest/rustls, and `rustls-platform-verifier` resolves roots through @@ -961,7 +963,10 @@ in # Order the container after the host CA generator, so the bind source # exists before nspawn sets the mount up. Without it a late CA fails the # container start outright rather than degrading. - systemd.services."container@${machine}" = caTrust.containerOrdering; + systemd.services."container@${machine}" = lib.mkMerge [ + caTrust.containerOrdering + (rootfsCheck.warnOnly machine) + ]; # Deliver the responder's store identity into the container before it # starts. A copy rather than a `bindMounts` entry: nixos-container refuses diff --git a/nix/host-modules/swarm-notify-router.nix b/nix/host-modules/swarm-notify-router.nix index e3ed74313..06e8d4927 100644 --- a/nix/host-modules/swarm-notify-router.nix +++ b/nix/host-modules/swarm-notify-router.nix @@ -44,6 +44,8 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8902; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -134,6 +136,7 @@ in # missing, so where this host mints the leaf the container waits for it. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; } ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 9c73bb81c..5504ca60c 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -173,6 +173,8 @@ let # that ever names the socket. hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The swarm's authelia is reached by its gateway name, whose leaf is # issued by the swarm services sub-CA — so this container needs the same # runtime CA trust every other consumer of a swarm-service name needs. @@ -706,7 +708,10 @@ in # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; + systemd.services."container@${cfg.machine}" = lib.mkMerge [ + caTrust.containerOrdering + (rootfsCheck.warnOnly cfg.machine) + ]; # A collector that is up but not listening on a socket the gateway names # answers every producer with a 502 and fails no unit. This one fails when diff --git a/nix/host-modules/swarm-review-classifier.nix b/nix/host-modules/swarm-review-classifier.nix index f6fd2d4a4..02a110a1d 100644 --- a/nix/host-modules/swarm-review-classifier.nix +++ b/nix/host-modules/swarm-review-classifier.nix @@ -57,6 +57,8 @@ let # The standard OTLP/HTTP port, on the container's loopback. forwarderOtlpPort = 4318; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -172,6 +174,7 @@ in # the unit does not exist. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 559fc1ef2..007e8a9d9 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -99,6 +99,8 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; + # The forwarder reaches the swarm collector and authelia by name over TLS; in # self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { @@ -290,6 +292,7 @@ in # ordering keeps the trust bundle's bind source there first. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering + (rootfsCheck.warnOnly cfg.machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 48730bb1b..04c849a37 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -198,6 +198,7 @@ let storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; + rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; in { # What the store IS from any hive's point of view, the name it answers on, @@ -728,6 +729,8 @@ in systemd.timers.${forwarderFetchUnit} = storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store"; + systemd.services."container@${cfg.machine}" = rootfsCheck.warnOnly cfg.machine; + containers.${cfg.machine} = { autoStart = true; ephemeral = false; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index c2c99f8ec..cfe315c8b 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -19,6 +19,7 @@ let ; }) hive + hiveWith hiveWithController runGroup forceServiceConfigs @@ -73,6 +74,13 @@ let gateway.openFirewall = true; }; + # A container the operator runs that hyperhive doesn't own, next to one it + # does (the classifier's). + withForeignContainer = hiveWith [ { containers.operator-box.config = { }; } ] { + deploy.bao.enable = true; + deploy.review-classifier.enable = true; + }; + # On the store's host, which mints the runner's store identity. withCi = hive { deploy.bao.enable = true; @@ -1497,6 +1505,105 @@ let && (sc.ProtectKernelLogs or false) && (sc.ProtectKernelModules or false); } + { + # Hyperhive's own declarative containers each run the root check from + # their own unit, warn-only. Every container in these fixtures is one of + # hyperhive's, so the names are read off the fixture: a container added + # without the check breaks the list, which is the point. The unit's own + # pre-start script is not a `hive-priv` entry, so the count is the check + # alone. + name = "every hyperhive container runs the root check warn-only from its own unit, and no hyperhive container is left out"; + ok = + let + checks = + m: name: + let + pre = m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]; + in + builtins.filter (lib.hasInfix "hive-priv") ( + map (p: builtins.unsafeDiscardStringContext (toString p)) pre + ); + carries = + m: name: + checks m name == [ + (builtins.unsafeDiscardStringContext "-${m.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}") + ]; + names = lib.attrNames allLocal.containers; + in + names == [ + "hive-forge" + "hive-matrix" + "swarm-attic" + "swarm-authelia" + "swarm-bao" + "swarm-builder" + "swarm-claude-login" + "swarm-grafana" + "swarm-nats" + "swarm-notify-router" + "swarm-otel" + "swarm-review-classifier" + "swarm-victorialogs" + "swarm-victoriametrics" + ] + && lib.all (carries allLocal) names + && withCi.containers ? swarm-ci-runner + && lib.all (carries withCi) (lib.attrNames withCi.containers) + # Control: a name with no unit finds nothing. + && checks allLocal "swarm-nope" == [ ]; + } + { + # Advisory means the start goes on whatever the check does, a crash or a + # missing binary included: systemd ignores the failure of a command + # prefixed `-`. The agents' strict line is the opposite and has no + # prefix; hive-priv renders it at runtime, and its own test + # (`the_rootfs_check_leads_the_service_section_…`) pins the bare + # `ExecStartPre=` there. + name = "every hyperhive container's root check is ignored on failure, not left to fail the start"; + ok = + let + hivePrivLines = + m: name: + builtins.filter (lib.hasInfix "hive-priv") ( + map (p: builtins.unsafeDiscardStringContext (toString p)) ( + m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ] + ) + ); + allLines = m: lib.concatMap (hivePrivLines m) (lib.attrNames m.containers); + ignored = lib.hasPrefix "-/"; + in + allLines allLocal != [ ] + && lib.length (allLines allLocal) == lib.length (lib.attrNames allLocal.containers) + && lib.all ignored (allLines allLocal) + && lib.all ignored (allLines withCi) + # Control: the test discriminates, a line without the prefix is not ignored. + && !(ignored "/nix/store/x-hyperhive/bin/hive-priv check-rootfs --warn-only x"); + } + { + # The operator's other containers are not hyperhive's to touch: a + # `containers.` it declares gets no check, while the classifier + # beside it does. The positive arm is what makes the absence mean + # something: the same lookup finds the classifier's. + name = "a container hyperhive does not declare gets no root check"; + ok = + let + checks = + name: + builtins.filter (lib.hasInfix "hive-priv") ( + map (p: builtins.unsafeDiscardStringContext (toString p)) ( + withForeignContainer.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ] + ) + ); + in + lib.attrNames withForeignContainer.containers == [ + "operator-box" + "swarm-bao" + "swarm-review-classifier" + ] + && lib.length (checks "swarm-review-classifier") == 1 + && lib.length (checks "swarm-bao") == 1 + && checks "operator-box" == [ ]; + } ] # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed # at one fires and fetches nothing. diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index b9f84ac54..fa10706f1 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -42,8 +42,8 @@ let # nixpkgs. `read-only.nix` makes any host module that defines top-level # `nixpkgs.config`/`overlays`/`hostPlatform` fail the fixture's eval instead # of being ignored. - hive = - extra: + hiveWith = + modules: extra: (nixosSystem { modules = [ (pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix") @@ -70,9 +70,13 @@ let swarm.hives.h1.domain = "h1.t.local"; } extra; } - ]; + ] + ++ modules; }).config; + # `hiveWith` with no extra modules: the host a fixture normally is. + hive = hiveWith [ ]; + # `hive` with the hive-controller tier on (hive-c0re, the agent containers, # `deploy.hive-controller.*`). Only a fixture whose claim reads that tier # takes this; every other claim is about the swarm services and evaluates @@ -299,7 +303,7 @@ let builtins.foldl' (acc: v: builtins.seq v acc) true vals; in { - inherit hive; + inherit hive hiveWith; inherit hiveWithController; inherit agent; inherit agentWith;