diff --git a/docs/agent-lifecycle/persistence.md b/docs/agent-lifecycle/persistence.md index 294a30a80..25be4475b 100644 --- a/docs/agent-lifecycle/persistence.md +++ b/docs/agent-lifecycle/persistence.md @@ -415,17 +415,6 @@ Destroying the container removes the drop-in and then its root; the build stays until the next collection. A drop-in written by an older hive-priv has no `ExecStartPre=` and no root, and the container starts as before. -**Hyperhive's platform containers: warn only.** The forge, the matrix server and -the swarm services (15 containers, declared by their own modules) run the same -check as `hive-priv check-rootfs --warn-only `, from `ExecStartPre=` on -their own `container@` unit. It evaluates every rule, logs each violation -as a warning, and exits 0, so the container starts either way. The line reads -`rootfs check (warn-only): : ` with journald's warning priority. -Strict refusal needs the live roots of all 15 to pass first, and the warnings -show which ones would not. A container the operator runs that hyperhive doesn't -declare gets no check at all: the modules name their own containers, and -nothing enumerates the host's `containers`. - **Disk use.** The copy shares blocks with the old root where it can: hive-priv clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and copies the data of a file the filesystem refuses to clone. Before it creates diff --git a/hive-priv/src/prestart.rs b/hive-priv/src/prestart.rs index b0a0683e2..5faccc751 100644 --- a/hive-priv/src/prestart.rs +++ b/hive-priv/src/prestart.rs @@ -3,17 +3,12 @@ //! nixpkgs' `container@.service` start script runs as host root against the //! container's own root, which is persistent and writable by the container. //! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a -//! link the container planted at any of the names the script touches. An agent -//! container's drop-in ([`crate::container_config`]) puts +//! link the container planted at any of the names the script touches. The +//! per-container drop-in ([`crate::container_config`]) puts //! `hive-priv check-rootfs ` ahead of that script as an //! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped //! while this runs, so nothing changes under the walk. //! -//! Hyperhive's own declarative containers (the forge, the matrix server, the -//! swarm services) run it as `check-rootfs --warn-only ` from their -//! own unit (`nix/host-modules/lib/container-rootfs-check.nix`): each violation -//! is logged at warning level and the start goes on. -//! //! Every component is looked at with `symlink_metadata`: a link is reported, //! never followed. Nothing is deleted or repaired, so whatever is there stays //! for the operator. @@ -32,15 +27,11 @@ use std::path::{Path, PathBuf}; use anyhow::{Context as _, Result, bail}; use crate::container_config::{container_rootfs, validate_container_system_name}; -use crate::shared::{partial_name, validate_name_chars}; +use crate::shared::partial_name; /// The sub-command the drop-in's `ExecStartPre=` names. pub(crate) const SUBCOMMAND: &str = "check-rootfs"; -/// The flag that turns a violation into a warning: the check runs in full and -/// exits 0. -const WARN_ONLY: &str = "--warn-only"; - /// The per-container nix profile directories on the host. const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container"; @@ -144,84 +135,33 @@ fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> { /// Handle `hive-priv ...` when the first argument names one. /// `None` means it is not a sub-command and the daemon starts. /// -/// `check-rootfs ` is strict: it prints one line to stderr and -/// returns exit code 1 on a violation, so the reason lands in the journal next -/// to the failed unit. `check-rootfs --warn-only ` prints every -/// violation as a warning and returns 0. +/// Prints one line to stderr and returns exit code 1 on a violation, so the +/// reason lands in the journal next to the failed unit. pub(crate) fn run(args: &[String]) -> Option { if args.first().map(String::as_str) != Some(SUBCOMMAND) { return None; } - let (container, warn_only) = match args { - [_, container] => (container, false), - [_, flag, container] if flag == WARN_ONLY => (container, true), - _ => { - eprintln!("usage: hive-priv {SUBCOMMAND} [{WARN_ONLY}] "); - return Some(1); - } + let result = match args { + [_, container] => check_container(container), + _ => Err(anyhow::anyhow!("usage: hive-priv {SUBCOMMAND} ")), }; - let found = found_in_container(container, warn_only); - let (code, lines) = report(container, &found, warn_only); - for line in lines { - eprintln!("{line}"); - } - Some(code) + Some(match result { + Ok(()) => 0, + Err(e) => { + eprintln!("hive-priv {SUBCOMMAND}: refusing to start: {e:#}"); + 1 + } + }) } -/// What a start would be refused for, on the host paths of `container`. -/// -/// Strict mode stops at the first violation, as the start would. Warn-only -/// lists them all, and counts a name it can't use as one more: it never fails -/// the start over its own input. -fn found_in_container(container: &str, warn_only: bool) -> Vec { - let paths = container_paths(container, warn_only); - match (paths, warn_only) { - (Ok((root, system)), true) => violations(&root, &system, Path::new(STORE)), - (Ok((root, system)), false) => check(&root, &system, Path::new(STORE)) - .err() - .map(|e| format!("{e:#}")) - .into_iter() - .collect(), - (Err(e), _) => vec![format!("{e:#}")], - } -} - -/// The container's root and its profile's `system` link on the host. -/// -/// An agent container is named `h-`. A warn-only check also takes the -/// platform containers' names, which are lowercase ascii, digits and hyphens. -/// Either way the name holds no `/` and is never `.` or `..`. -fn container_paths(container: &str, warn_only: bool) -> Result<(PathBuf, PathBuf)> { - if warn_only { - validate_name_chars(container)?; - } else { - validate_container_system_name(container)?; - } - Ok(( - container_rootfs(container), - Path::new(PROFILES_ROOT).join(container).join("system"), - )) -} - -/// The exit code and the stderr lines for the violations `found`. -/// -/// Warn-only lines carry the `<4>` prefix, which journald reads as the -/// warning priority, so a log query can select them by severity. -fn report(container: &str, found: &[String], warn_only: bool) -> (i32, Vec) { - if warn_only { - let lines = found - .iter() - .map(|v| format!("<4>rootfs check (warn-only): {container}: {v}")) - .collect(); - return (0, lines); - } - match found.first() { - Some(v) => ( - 1, - vec![format!("hive-priv {SUBCOMMAND}: refusing to start: {v}")], - ), - None => (0, Vec::new()), - } +/// [`check`] on the host paths of `container`. +fn check_container(container: &str) -> Result<()> { + validate_container_system_name(container)?; + check( + &container_rootfs(container), + &Path::new(PROFILES_ROOT).join(container).join("system"), + Path::new(STORE), + ) } /// Check the paths the start script touches as root, returning an error that @@ -233,36 +173,14 @@ fn report(container: &str, found: &[String], warn_only: bool) -> (i32, Vec Result<()> { - match violations(root, system, store).into_iter().next() { - None => Ok(()), - Some(first) => bail!("{first}"), - } -} - -/// Every path under the rules of [`check`] that holds something it must not, -/// in the order [`check`] looks at them, each message once. -fn violations(root: &Path, system: &Path, store: &Path) -> Vec { - let mut found = Vec::new(); for dir in DIRS { - note(&mut found, check_dir(root, dir)); + check_dir(root, dir)?; } for file in REGULAR_ONLY { - note(&mut found, check_file(root, file, false)); - } - note(&mut found, check_file(root, OS_RELEASE, true)); - note(&mut found, check_system(system, store)); - found -} - -/// Add the message of an error to `found`, unless it is already there: a link -/// at `usr` is the finding for `usr/bin` too. -fn note(found: &mut Vec, result: Result<()>) { - if let Err(e) = result { - let message = format!("{e:#}"); - if !found.contains(&message) { - found.push(message); - } + check_file(root, file, false)?; } + check_file(root, OS_RELEASE, true)?; + check_system(system, store) } /// `symlink_metadata`, with `None` for a path that does not exist. The one @@ -367,7 +285,7 @@ mod tests { use std::os::unix::fs::symlink; use std::path::PathBuf; - use super::{check, container_paths, pin_binary_in, report, run, unpin_binary_in, violations}; + use super::{check, pin_binary_in, run, unpin_binary_in}; use crate::shared::test_support::{entries, scratch}; /// A scratch container: root, the profile link's path and a store with one @@ -690,133 +608,16 @@ mod tests { assert!(msg.contains("left alone"), "{msg}"); assert!(roots.join("h-iris/keep").exists()); } - #[test] fn only_the_check_rootfs_sub_command_is_taken() { - let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::>(); assert_eq!(run(&[]), None); - assert_eq!(run(&args(&["serve"])), None); - // Wrong arity, an unknown flag and a name that is not an agent - // container are refused, whatever is on disk. - assert_eq!(run(&args(&["check-rootfs"])), Some(1)); - assert_eq!(run(&args(&["check-rootfs", "not-hive"])), Some(1)); - assert_eq!(run(&args(&["check-rootfs", "--strict", "h-iris"])), Some(1)); - assert_eq!(run(&args(&["check-rootfs", "--warn-only"])), Some(1)); - } - - #[test] - fn a_warn_only_run_exits_zero_even_for_a_name_it_cannot_use() { - let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::>(); + assert_eq!(run(&["serve".to_string()]), None); + // Wrong arity and a name that is not an agent container are refused, + // whatever is on disk. + assert_eq!(run(&["check-rootfs".to_string()]), Some(1)); assert_eq!( - run(&args(&["check-rootfs", "--warn-only", "no-such-container"])), - Some(0) + run(&["check-rootfs".to_string(), "not-hive".to_string()]), + Some(1) ); - assert_eq!( - run(&args(&["check-rootfs", "--warn-only", "../etc"])), - Some(0) - ); - } - - #[test] - fn strict_takes_agent_names_and_warn_only_takes_plain_ones() { - assert!(container_paths("h-iris", false).is_ok()); - assert!(container_paths("hive-forge", false).is_err()); - for name in ["hive-forge", "swarm-review-classifier", "h-iris"] { - let (root, system) = container_paths(name, true).unwrap(); - assert_eq!( - root, - PathBuf::from(format!("/var/lib/nixos-containers/{name}")) - ); - assert_eq!( - system, - PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system")) - ); - } - for name in [ - "", - ".", - "..", - "../h-iris", - "a/b", - "/etc", - "H-iris", - "a.b", - "a b", - ] { - assert!( - container_paths(name, true).is_err(), - "{name:?} was accepted" - ); - assert!( - container_paths(name, false).is_err(), - "{name:?} was accepted" - ); - } - } - - /// A clean root with a link at `root` and another at `etc/resolv.conf`. - fn two_violations() -> Fixture { - let f = Fixture::clean(); - std::fs::remove_dir_all(f.root.join("root")).unwrap(); - symlink(victim(), f.root.join("root")).unwrap(); - std::fs::remove_file(f.root.join("etc/resolv.conf")).unwrap(); - symlink("/etc/hosts", f.root.join("etc/resolv.conf")).unwrap(); - f - } - - fn found(f: &Fixture) -> Vec { - violations(&f.root, &f.system, &f.store) - } - - #[test] - fn warn_only_logs_every_violation_at_warning_level_and_exits_zero() { - let f = two_violations(); - let found = found(&f); - assert_eq!(found.len(), 2, "{found:?}"); - let (code, lines) = report("hive-forge", &found, true); - assert_eq!(code, 0); - assert_eq!(lines.len(), 2, "{lines:?}"); - for line in &lines { - assert!( - line.starts_with("<4>rootfs check (warn-only): hive-forge: "), - "{line}" - ); - } - assert!(lines[0].contains("root is a symlink"), "{lines:?}"); - assert!(lines[1].contains("resolv.conf is a symlink"), "{lines:?}"); - } - - #[test] - fn strict_refuses_with_the_first_violation_only() { - let f = two_violations(); - let found = found(&f); - let (code, lines) = report("h-iris", &found, false); - assert_eq!(code, 1); - assert_eq!(lines.len(), 1, "{lines:?}"); - assert!( - lines[0].starts_with("hive-priv check-rootfs: refusing to start: "), - "{lines:?}" - ); - assert!(lines[0].contains("root is a symlink"), "{lines:?}"); - // The rules are the same: the strict refusal is `check`'s first error. - assert_eq!(f.refused(), found[0]); - } - - #[test] - fn a_clean_root_prints_nothing_in_either_mode() { - let f = Fixture::clean(); - assert!(found(&f).is_empty()); - assert_eq!(report("hive-forge", &[], true), (0, Vec::new())); - assert_eq!(report("h-iris", &[], false), (0, Vec::new())); - } - - #[test] - fn one_link_is_reported_once_though_two_paths_run_through_it() { - let f = Fixture::clean(); - std::fs::remove_dir_all(f.root.join("usr")).unwrap(); - symlink(victim(), f.root.join("usr")).unwrap(); - let found = found(&f); - assert_eq!(found.len(), 1, "{found:?}"); - assert!(found[0].contains("usr is a symlink"), "{found:?}"); } } diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 565bbbc37..647ea9954 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -78,8 +78,6 @@ let deployCfg.bao.forgePublisherClientCertFile != null && deployCfg.bao.forgePublisherClientKeyFile != null; - rootfsCheck = import ../lib/container-rootfs-check.nix { inherit config; }; - caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # The container's journal forwarder pushes to its own receiver on the swarm @@ -473,7 +471,6 @@ in # ../swarm-builder.nix's does. systemd.services."container@hive-forge" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly "hive-forge") { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index c77a847f2..d691cbbac 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -13,8 +13,6 @@ let # the client is pointed at the server name instead. hyperhiveDomain = config.services.hyperhive.domain; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # Same runtime→build-time bridge hive-forge and swarm-ci-runner cross: # binds the hive trust bundle (which folds in the store's root) into the # container and orders the container after `hive-tls-ca.service`. The @@ -1098,7 +1096,6 @@ in systemd.services."container@hive-matrix" = lib.mkMerge [ { after = [ "dnsmasq.service" ]; } caTrust.containerOrdering - (rootfsCheck.warnOnly "hive-matrix") ]; }; } diff --git a/nix/host-modules/lib/container-rootfs-check.nix b/nix/host-modules/lib/container-rootfs-check.nix deleted file mode 100644 index 5922c80a7..000000000 --- a/nix/host-modules/lib/container-rootfs-check.nix +++ /dev/null @@ -1,34 +0,0 @@ -# The pre-start check of a container's root, for one of hyperhive's own -# declarative containers. -# -# nixpkgs' start script runs as host root against the container's own root, -# which is persistent and writable by the container, and its `mkdir`, `chmod`, -# `touch` and `cp` follow links the container planted there. `hive-priv -# check-rootfs` looks at the paths the script touches (rules in -# `hive-priv/src/prestart.rs`). With `--warn-only` it logs each violation at -# warning level and lets the start go on. The line carries systemd's `-` -# prefix, so a crash or a failed exec of the check cannot fail the start either. -# The agent containers run it strict -# from their per-agent drop-in (hive-priv's `WriteResourceLimits`). -# -# Pure function, not a NixOS module. A container's module calls it with its -# own name and folds the result into its unit: -# -# rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; -# # systemd.services."container@${machine}" = lib.mkMerge [ (rootfsCheck.warnOnly machine) … ]; -# -# The name list is the callers', never `config.containers`: that also holds -# containers the operator runs that hyperhive doesn't own, and those get -# nothing. nixpkgs renders each declarative container as its own -# `container@` unit (`nixos-containers.nix`, the `mapAttrsToList` that -# builds `container@${name}`), so a setting on that unit reaches that -# container only. -{ config }: -{ - # Attributes for `systemd.services."container@"`. - warnOnly = name: { - serviceConfig.ExecStartPre = [ - "-${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}" - ]; - }; -} diff --git a/nix/host-modules/swarm-attic.nix b/nix/host-modules/swarm-attic.nix index 9b6f0477a..de203d2e1 100644 --- a/nix/host-modules/swarm-attic.nix +++ b/nix/host-modules/swarm-attic.nix @@ -114,8 +114,6 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The forwarder reaches the swarm collector and authelia by name over TLS; in # self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { @@ -228,7 +226,6 @@ in # CA ordering keeps the trust bundle's bind source there first. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 39783827f..2e5911775 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -183,8 +183,6 @@ let refreshConsumer = import ./lib/refresh-consumer.nix { }; storeSeededSecret = import ./lib/store-seeded-secret.nix { }; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - caTrust = import ./lib/hive-ca-trust.nix { inherit lib gatewayCfg; tlsCfg = deployCfg.hive-controller.tls; @@ -972,7 +970,6 @@ in # not be up. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly cfg.machine) (lib.mkIf publishes { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-bao.nix b/nix/host-modules/swarm-bao.nix index 9e6df0ea0..dd5b1bab7 100644 --- a/nix/host-modules/swarm-bao.nix +++ b/nix/host-modules/swarm-bao.nix @@ -1689,8 +1689,6 @@ let forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}"; autheliaCfg = hyperhiveCfg.swarm.authelia; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # Outbound TLS for that collector: it dials the swarm collector's gateway # name and authelia's token endpoint, both leaves under the store's runtime # pki root, which cannot be baked into a derivation. Same helper, same two @@ -4093,7 +4091,6 @@ in # cannot start is a store that is down. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly cfg.machine) { after = [ "systemd-tmpfiles-resetup.service" ]; } ]; diff --git a/nix/host-modules/swarm-builder.nix b/nix/host-modules/swarm-builder.nix index 70234fca2..1fb12bd72 100644 --- a/nix/host-modules/swarm-builder.nix +++ b/nix/host-modules/swarm-builder.nix @@ -52,8 +52,6 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8892; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The daemon reaches authelia, the store and the queue by name over TLS, and # `git` reaches the forge; in self-signed mode their leaves chain to the hive # CA. @@ -167,7 +165,6 @@ in # the unit does not exist. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-ci-runner.nix b/nix/host-modules/swarm-ci-runner.nix index a67cd9c56..690b0348b 100644 --- a/nix/host-modules/swarm-ci-runner.nix +++ b/nix/host-modules/swarm-ci-runner.nix @@ -43,8 +43,6 @@ let # below compares the two, so both come from here. forgeUrl = "https://${forgeCfg.domain}"; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The forge's gateway serves a leaf from the store's `pki` mount in # self-signed mode. # The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store, @@ -331,7 +329,6 @@ in # minting unit in and waits for it, as ./swarm-builder.nix's does. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-claude-login.nix b/nix/host-modules/swarm-claude-login.nix index c7d3b7826..1c75147f4 100644 --- a/nix/host-modules/swarm-claude-login.nix +++ b/nix/host-modules/swarm-claude-login.nix @@ -54,8 +54,6 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8893; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -148,7 +146,6 @@ in # missing, so where this host mints the leaf the container waits for it. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; } ]; diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 49853a4ee..a5bbb2f43 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -24,8 +24,6 @@ let baoCfg = hyperhiveCfg.swarm.bao; baoDeploy = deployCfg.bao; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - caTrust = import ./lib/hive-ca-trust.nix { inherit lib gatewayCfg; tlsCfg = deployCfg.hive-controller.tls; @@ -688,7 +686,6 @@ in # ./swarm-builder.nix's does. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly cfg.machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 845c838c5..45b857e16 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -33,8 +33,6 @@ let tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; gatewayCfg = config.services.hyperhive.gateway; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; # The responder introspects authelia over https BY NAME. Its HTTP client is # reqwest/rustls, and `rustls-platform-verifier` resolves roots through @@ -963,10 +961,7 @@ in # Order the container after the host CA generator, so the bind source # exists before nspawn sets the mount up. Without it a late CA fails the # container start outright rather than degrading. - systemd.services."container@${machine}" = lib.mkMerge [ - caTrust.containerOrdering - (rootfsCheck.warnOnly machine) - ]; + systemd.services."container@${machine}" = caTrust.containerOrdering; # Deliver the responder's store identity into the container before it # starts. A copy rather than a `bindMounts` entry: nixos-container refuses diff --git a/nix/host-modules/swarm-notify-router.nix b/nix/host-modules/swarm-notify-router.nix index 06e8d4927..e3ed74313 100644 --- a/nix/host-modules/swarm-notify-router.nix +++ b/nix/host-modules/swarm-notify-router.nix @@ -44,8 +44,6 @@ let # containers' forwarders all the same. forwarderTelemetryPort = 8902; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -136,7 +134,6 @@ in # missing, so where this host mints the leaf the container waits for it. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; } ]; diff --git a/nix/host-modules/swarm-otel.nix b/nix/host-modules/swarm-otel.nix index 5504ca60c..9c73bb81c 100644 --- a/nix/host-modules/swarm-otel.nix +++ b/nix/host-modules/swarm-otel.nix @@ -173,8 +173,6 @@ let # that ever names the socket. hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The swarm's authelia is reached by its gateway name, whose leaf is # issued by the swarm services sub-CA — so this container needs the same # runtime CA trust every other consumer of a swarm-service name needs. @@ -708,10 +706,7 @@ in # The CA bind source is written at runtime by a host unit, so the # container has to start after it — otherwise nspawn sets up a mount # over a file that does not exist yet. - systemd.services."container@${cfg.machine}" = lib.mkMerge [ - caTrust.containerOrdering - (rootfsCheck.warnOnly cfg.machine) - ]; + systemd.services."container@${cfg.machine}" = caTrust.containerOrdering; # A collector that is up but not listening on a socket the gateway names # answers every producer with a 502 and fails no unit. This one fails when diff --git a/nix/host-modules/swarm-review-classifier.nix b/nix/host-modules/swarm-review-classifier.nix index 02a110a1d..f6fd2d4a4 100644 --- a/nix/host-modules/swarm-review-classifier.nix +++ b/nix/host-modules/swarm-review-classifier.nix @@ -57,8 +57,6 @@ let # The standard OTLP/HTTP port, on the container's loopback. forwarderOtlpPort = 4318; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The daemon reaches authelia, the store and the queue by name over TLS, and # in self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; }; @@ -174,7 +172,6 @@ in # the unit does not exist. systemd.services."container@${machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-victorialogs.nix b/nix/host-modules/swarm-victorialogs.nix index 007e8a9d9..559fc1ef2 100644 --- a/nix/host-modules/swarm-victorialogs.nix +++ b/nix/host-modules/swarm-victorialogs.nix @@ -99,8 +99,6 @@ let # one word. oneWord = path: builtins.match ".*[[:space:]].*" path == null; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; - # The forwarder reaches the swarm collector and authelia by name over TLS; in # self-signed mode their leaves chain to the hive CA. caTrust = import ./lib/hive-ca-trust.nix { @@ -292,7 +290,6 @@ in # ordering keeps the trust bundle's bind source there first. systemd.services."container@${cfg.machine}" = lib.mkMerge [ caTrust.containerOrdering - (rootfsCheck.warnOnly cfg.machine) { after = [ "swarm-bao-pki.service" ]; wants = [ "swarm-bao-pki.service" ]; diff --git a/nix/host-modules/swarm-victoriametrics.nix b/nix/host-modules/swarm-victoriametrics.nix index 04c849a37..48730bb1b 100644 --- a/nix/host-modules/swarm-victoriametrics.nix +++ b/nix/host-modules/swarm-victoriametrics.nix @@ -198,7 +198,6 @@ let storeRefreshTimer = import ./lib/store-refresh-timer.nix { }; atomicWriteSecret = import ./lib/atomic-write-secret.nix { }; refreshConsumer = import ./lib/refresh-consumer.nix { }; - rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; }; in { # What the store IS from any hive's point of view, the name it answers on, @@ -729,8 +728,6 @@ in systemd.timers.${forwarderFetchUnit} = storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store"; - systemd.services."container@${cfg.machine}" = rootfsCheck.warnOnly cfg.machine; - containers.${cfg.machine} = { autoStart = true; ephemeral = false; diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix index cfe315c8b..c2c99f8ec 100644 --- a/nix/module-eval/core-toggle.nix +++ b/nix/module-eval/core-toggle.nix @@ -19,7 +19,6 @@ let ; }) hive - hiveWith hiveWithController runGroup forceServiceConfigs @@ -74,13 +73,6 @@ let gateway.openFirewall = true; }; - # A container the operator runs that hyperhive doesn't own, next to one it - # does (the classifier's). - withForeignContainer = hiveWith [ { containers.operator-box.config = { }; } ] { - deploy.bao.enable = true; - deploy.review-classifier.enable = true; - }; - # On the store's host, which mints the runner's store identity. withCi = hive { deploy.bao.enable = true; @@ -1505,105 +1497,6 @@ let && (sc.ProtectKernelLogs or false) && (sc.ProtectKernelModules or false); } - { - # Hyperhive's own declarative containers each run the root check from - # their own unit, warn-only. Every container in these fixtures is one of - # hyperhive's, so the names are read off the fixture: a container added - # without the check breaks the list, which is the point. The unit's own - # pre-start script is not a `hive-priv` entry, so the count is the check - # alone. - name = "every hyperhive container runs the root check warn-only from its own unit, and no hyperhive container is left out"; - ok = - let - checks = - m: name: - let - pre = m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]; - in - builtins.filter (lib.hasInfix "hive-priv") ( - map (p: builtins.unsafeDiscardStringContext (toString p)) pre - ); - carries = - m: name: - checks m name == [ - (builtins.unsafeDiscardStringContext "-${m.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}") - ]; - names = lib.attrNames allLocal.containers; - in - names == [ - "hive-forge" - "hive-matrix" - "swarm-attic" - "swarm-authelia" - "swarm-bao" - "swarm-builder" - "swarm-claude-login" - "swarm-grafana" - "swarm-nats" - "swarm-notify-router" - "swarm-otel" - "swarm-review-classifier" - "swarm-victorialogs" - "swarm-victoriametrics" - ] - && lib.all (carries allLocal) names - && withCi.containers ? swarm-ci-runner - && lib.all (carries withCi) (lib.attrNames withCi.containers) - # Control: a name with no unit finds nothing. - && checks allLocal "swarm-nope" == [ ]; - } - { - # Advisory means the start goes on whatever the check does, a crash or a - # missing binary included: systemd ignores the failure of a command - # prefixed `-`. The agents' strict line is the opposite and has no - # prefix; hive-priv renders it at runtime, and its own test - # (`the_rootfs_check_leads_the_service_section_…`) pins the bare - # `ExecStartPre=` there. - name = "every hyperhive container's root check is ignored on failure, not left to fail the start"; - ok = - let - hivePrivLines = - m: name: - builtins.filter (lib.hasInfix "hive-priv") ( - map (p: builtins.unsafeDiscardStringContext (toString p)) ( - m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ] - ) - ); - allLines = m: lib.concatMap (hivePrivLines m) (lib.attrNames m.containers); - ignored = lib.hasPrefix "-/"; - in - allLines allLocal != [ ] - && lib.length (allLines allLocal) == lib.length (lib.attrNames allLocal.containers) - && lib.all ignored (allLines allLocal) - && lib.all ignored (allLines withCi) - # Control: the test discriminates, a line without the prefix is not ignored. - && !(ignored "/nix/store/x-hyperhive/bin/hive-priv check-rootfs --warn-only x"); - } - { - # The operator's other containers are not hyperhive's to touch: a - # `containers.` it declares gets no check, while the classifier - # beside it does. The positive arm is what makes the absence mean - # something: the same lookup finds the classifier's. - name = "a container hyperhive does not declare gets no root check"; - ok = - let - checks = - name: - builtins.filter (lib.hasInfix "hive-priv") ( - map (p: builtins.unsafeDiscardStringContext (toString p)) ( - withForeignContainer.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ] - ) - ); - in - lib.attrNames withForeignContainer.containers == [ - "operator-box" - "swarm-bao" - "swarm-review-classifier" - ] - && lib.length (checks "swarm-review-classifier") == 1 - && lib.length (checks "swarm-bao") == 1 - && checks "operator-box" == [ ]; - } ] # Starting a unit that stays `active (exited)` is a no-op, so a timer aimed # at one fires and fetches nothing. diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix index fa10706f1..b9f84ac54 100644 --- a/nix/module-eval/lib.nix +++ b/nix/module-eval/lib.nix @@ -42,8 +42,8 @@ let # nixpkgs. `read-only.nix` makes any host module that defines top-level # `nixpkgs.config`/`overlays`/`hostPlatform` fail the fixture's eval instead # of being ignored. - hiveWith = - modules: extra: + hive = + extra: (nixosSystem { modules = [ (pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix") @@ -70,13 +70,9 @@ let swarm.hives.h1.domain = "h1.t.local"; } extra; } - ] - ++ modules; + ]; }).config; - # `hiveWith` with no extra modules: the host a fixture normally is. - hive = hiveWith [ ]; - # `hive` with the hive-controller tier on (hive-c0re, the agent containers, # `deploy.hive-controller.*`). Only a fixture whose claim reads that tier # takes this; every other claim is about the swarm services and evaluates @@ -303,7 +299,7 @@ let builtins.foldl' (acc: v: builtins.seq v acc) true vals; in { - inherit hive hiveWith; + inherit hive; inherit hiveWithController; inherit agent; inherit agentWith;