Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

...
20 changed files with 444 additions and 40 deletions

View file

@ -415,6 +415,17 @@ Destroying the container removes the drop-in and then its root; the build stays
until the next collection. A drop-in written by an older hive-priv has no
`ExecStartPre=` and no root, and the container starts as before.
**Hyperhive's platform containers: warn only.** The forge, the matrix server and
the swarm services (15 containers, declared by their own modules) run the same
check as `hive-priv check-rootfs --warn-only <name>`, from `ExecStartPre=` on
their own `container@<name>` unit. It evaluates every rule, logs each violation
as a warning, and exits 0, so the container starts either way. The line reads
`rootfs check (warn-only): <name>: <message>` with journald's warning priority.
Strict refusal needs the live roots of all 15 to pass first, and the warnings
show which ones would not. A container the operator runs that hyperhive doesn't
declare gets no check at all: the modules name their own containers, and
nothing enumerates the host's `containers`.
**Disk use.** The copy shares blocks with the old root where it can: hive-priv
clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
copies the data of a file the filesystem refuses to clone. Before it creates

View file

@ -3,12 +3,17 @@
//! nixpkgs' `container@.service` start script runs as host root against the
//! container's own root, which is persistent and writable by the container.
//! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a
//! link the container planted at any of the names the script touches. The
//! per-container drop-in ([`crate::container_config`]) puts
//! link the container planted at any of the names the script touches. An agent
//! container's drop-in ([`crate::container_config`]) puts
//! `hive-priv check-rootfs <container>` ahead of that script as an
//! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped
//! while this runs, so nothing changes under the walk.
//!
//! Hyperhive's own declarative containers (the forge, the matrix server, the
//! swarm services) run it as `check-rootfs --warn-only <container>` from their
//! own unit (`nix/host-modules/lib/container-rootfs-check.nix`): each violation
//! is logged at warning level and the start goes on.
//!
//! Every component is looked at with `symlink_metadata`: a link is reported,
//! never followed. Nothing is deleted or repaired, so whatever is there stays
//! for the operator.
@ -27,11 +32,15 @@ use std::path::{Path, PathBuf};
use anyhow::{Context as _, Result, bail};
use crate::container_config::{container_rootfs, validate_container_system_name};
use crate::shared::partial_name;
use crate::shared::{partial_name, validate_name_chars};
/// The sub-command the drop-in's `ExecStartPre=` names.
pub(crate) const SUBCOMMAND: &str = "check-rootfs";
/// The flag that turns a violation into a warning: the check runs in full and
/// exits 0.
const WARN_ONLY: &str = "--warn-only";
/// The per-container nix profile directories on the host.
const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container";
@ -135,33 +144,84 @@ fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> {
/// Handle `hive-priv <subcommand> ...` when the first argument names one.
/// `None` means it is not a sub-command and the daemon starts.
///
/// Prints one line to stderr and returns exit code 1 on a violation, so the
/// reason lands in the journal next to the failed unit.
/// `check-rootfs <container>` is strict: it prints one line to stderr and
/// returns exit code 1 on a violation, so the reason lands in the journal next
/// to the failed unit. `check-rootfs --warn-only <container>` prints every
/// violation as a warning and returns 0.
pub(crate) fn run(args: &[String]) -> Option<i32> {
if args.first().map(String::as_str) != Some(SUBCOMMAND) {
return None;
}
let result = match args {
[_, container] => check_container(container),
_ => Err(anyhow::anyhow!("usage: hive-priv {SUBCOMMAND} <container>")),
};
Some(match result {
Ok(()) => 0,
Err(e) => {
eprintln!("hive-priv {SUBCOMMAND}: refusing to start: {e:#}");
1
let (container, warn_only) = match args {
[_, container] => (container, false),
[_, flag, container] if flag == WARN_ONLY => (container, true),
_ => {
eprintln!("usage: hive-priv {SUBCOMMAND} [{WARN_ONLY}] <container>");
return Some(1);
}
})
};
let found = found_in_container(container, warn_only);
let (code, lines) = report(container, &found, warn_only);
for line in lines {
eprintln!("{line}");
}
Some(code)
}
/// [`check`] on the host paths of `container`.
fn check_container(container: &str) -> Result<()> {
validate_container_system_name(container)?;
check(
&container_rootfs(container),
&Path::new(PROFILES_ROOT).join(container).join("system"),
Path::new(STORE),
)
/// What a start would be refused for, on the host paths of `container`.
///
/// Strict mode stops at the first violation, as the start would. Warn-only
/// lists them all, and counts a name it can't use as one more: it never fails
/// the start over its own input.
fn found_in_container(container: &str, warn_only: bool) -> Vec<String> {
let paths = container_paths(container, warn_only);
match (paths, warn_only) {
(Ok((root, system)), true) => violations(&root, &system, Path::new(STORE)),
(Ok((root, system)), false) => check(&root, &system, Path::new(STORE))
.err()
.map(|e| format!("{e:#}"))
.into_iter()
.collect(),
(Err(e), _) => vec![format!("{e:#}")],
}
}
/// The container's root and its profile's `system` link on the host.
///
/// An agent container is named `h-<agent>`. A warn-only check also takes the
/// platform containers' names, which are lowercase ascii, digits and hyphens.
/// Either way the name holds no `/` and is never `.` or `..`.
fn container_paths(container: &str, warn_only: bool) -> Result<(PathBuf, PathBuf)> {
if warn_only {
validate_name_chars(container)?;
} else {
validate_container_system_name(container)?;
}
Ok((
container_rootfs(container),
Path::new(PROFILES_ROOT).join(container).join("system"),
))
}
/// The exit code and the stderr lines for the violations `found`.
///
/// Warn-only lines carry the `<4>` prefix, which journald reads as the
/// warning priority, so a log query can select them by severity.
fn report(container: &str, found: &[String], warn_only: bool) -> (i32, Vec<String>) {
if warn_only {
let lines = found
.iter()
.map(|v| format!("<4>rootfs check (warn-only): {container}: {v}"))
.collect();
return (0, lines);
}
match found.first() {
Some(v) => (
1,
vec![format!("hive-priv {SUBCOMMAND}: refusing to start: {v}")],
),
None => (0, Vec::new()),
}
}
/// Check the paths the start script touches as root, returning an error that
@ -173,14 +233,36 @@ fn check_container(container: &str) -> Result<()> {
/// A path that does not exist passes: a first start has nothing yet, and the
/// script creates it.
fn check(root: &Path, system: &Path, store: &Path) -> Result<()> {
match violations(root, system, store).into_iter().next() {
None => Ok(()),
Some(first) => bail!("{first}"),
}
}
/// Every path under the rules of [`check`] that holds something it must not,
/// in the order [`check`] looks at them, each message once.
fn violations(root: &Path, system: &Path, store: &Path) -> Vec<String> {
let mut found = Vec::new();
for dir in DIRS {
check_dir(root, dir)?;
note(&mut found, check_dir(root, dir));
}
for file in REGULAR_ONLY {
check_file(root, file, false)?;
note(&mut found, check_file(root, file, false));
}
note(&mut found, check_file(root, OS_RELEASE, true));
note(&mut found, check_system(system, store));
found
}
/// Add the message of an error to `found`, unless it is already there: a link
/// at `usr` is the finding for `usr/bin` too.
fn note(found: &mut Vec<String>, result: Result<()>) {
if let Err(e) = result {
let message = format!("{e:#}");
if !found.contains(&message) {
found.push(message);
}
}
check_file(root, OS_RELEASE, true)?;
check_system(system, store)
}
/// `symlink_metadata`, with `None` for a path that does not exist. The one
@ -285,7 +367,7 @@ mod tests {
use std::os::unix::fs::symlink;
use std::path::PathBuf;
use super::{check, pin_binary_in, run, unpin_binary_in};
use super::{check, container_paths, pin_binary_in, report, run, unpin_binary_in, violations};
use crate::shared::test_support::{entries, scratch};
/// A scratch container: root, the profile link's path and a store with one
@ -608,16 +690,133 @@ mod tests {
assert!(msg.contains("left alone"), "{msg}");
assert!(roots.join("h-iris/keep").exists());
}
#[test]
fn only_the_check_rootfs_sub_command_is_taken() {
let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::<Vec<_>>();
assert_eq!(run(&[]), None);
assert_eq!(run(&["serve".to_string()]), None);
// Wrong arity and a name that is not an agent container are refused,
// whatever is on disk.
assert_eq!(run(&["check-rootfs".to_string()]), Some(1));
assert_eq!(run(&args(&["serve"])), None);
// Wrong arity, an unknown flag and a name that is not an agent
// container are refused, whatever is on disk.
assert_eq!(run(&args(&["check-rootfs"])), Some(1));
assert_eq!(run(&args(&["check-rootfs", "not-hive"])), Some(1));
assert_eq!(run(&args(&["check-rootfs", "--strict", "h-iris"])), Some(1));
assert_eq!(run(&args(&["check-rootfs", "--warn-only"])), Some(1));
}
#[test]
fn a_warn_only_run_exits_zero_even_for_a_name_it_cannot_use() {
let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::<Vec<_>>();
assert_eq!(
run(&["check-rootfs".to_string(), "not-hive".to_string()]),
Some(1)
run(&args(&["check-rootfs", "--warn-only", "no-such-container"])),
Some(0)
);
assert_eq!(
run(&args(&["check-rootfs", "--warn-only", "../etc"])),
Some(0)
);
}
#[test]
fn strict_takes_agent_names_and_warn_only_takes_plain_ones() {
assert!(container_paths("h-iris", false).is_ok());
assert!(container_paths("hive-forge", false).is_err());
for name in ["hive-forge", "swarm-review-classifier", "h-iris"] {
let (root, system) = container_paths(name, true).unwrap();
assert_eq!(
root,
PathBuf::from(format!("/var/lib/nixos-containers/{name}"))
);
assert_eq!(
system,
PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system"))
);
}
for name in [
"",
".",
"..",
"../h-iris",
"a/b",
"/etc",
"H-iris",
"a.b",
"a b",
] {
assert!(
container_paths(name, true).is_err(),
"{name:?} was accepted"
);
assert!(
container_paths(name, false).is_err(),
"{name:?} was accepted"
);
}
}
/// A clean root with a link at `root` and another at `etc/resolv.conf`.
fn two_violations() -> Fixture {
let f = Fixture::clean();
std::fs::remove_dir_all(f.root.join("root")).unwrap();
symlink(victim(), f.root.join("root")).unwrap();
std::fs::remove_file(f.root.join("etc/resolv.conf")).unwrap();
symlink("/etc/hosts", f.root.join("etc/resolv.conf")).unwrap();
f
}
fn found(f: &Fixture) -> Vec<String> {
violations(&f.root, &f.system, &f.store)
}
#[test]
fn warn_only_logs_every_violation_at_warning_level_and_exits_zero() {
let f = two_violations();
let found = found(&f);
assert_eq!(found.len(), 2, "{found:?}");
let (code, lines) = report("hive-forge", &found, true);
assert_eq!(code, 0);
assert_eq!(lines.len(), 2, "{lines:?}");
for line in &lines {
assert!(
line.starts_with("<4>rootfs check (warn-only): hive-forge: "),
"{line}"
);
}
assert!(lines[0].contains("root is a symlink"), "{lines:?}");
assert!(lines[1].contains("resolv.conf is a symlink"), "{lines:?}");
}
#[test]
fn strict_refuses_with_the_first_violation_only() {
let f = two_violations();
let found = found(&f);
let (code, lines) = report("h-iris", &found, false);
assert_eq!(code, 1);
assert_eq!(lines.len(), 1, "{lines:?}");
assert!(
lines[0].starts_with("hive-priv check-rootfs: refusing to start: "),
"{lines:?}"
);
assert!(lines[0].contains("root is a symlink"), "{lines:?}");
// The rules are the same: the strict refusal is `check`'s first error.
assert_eq!(f.refused(), found[0]);
}
#[test]
fn a_clean_root_prints_nothing_in_either_mode() {
let f = Fixture::clean();
assert!(found(&f).is_empty());
assert_eq!(report("hive-forge", &[], true), (0, Vec::new()));
assert_eq!(report("h-iris", &[], false), (0, Vec::new()));
}
#[test]
fn one_link_is_reported_once_though_two_paths_run_through_it() {
let f = Fixture::clean();
std::fs::remove_dir_all(f.root.join("usr")).unwrap();
symlink(victim(), f.root.join("usr")).unwrap();
let found = found(&f);
assert_eq!(found.len(), 1, "{found:?}");
assert!(found[0].contains("usr is a symlink"), "{found:?}");
}
}

View file

@ -78,6 +78,8 @@ let
deployCfg.bao.forgePublisherClientCertFile != null
&& deployCfg.bao.forgePublisherClientKeyFile != null;
rootfsCheck = import ../lib/container-rootfs-check.nix { inherit config; };
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# The container's journal forwarder pushes to its own receiver on the swarm
@ -471,6 +473,7 @@ in
# ../swarm-builder.nix's does.
systemd.services."container@hive-forge" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly "hive-forge")
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -13,6 +13,8 @@ let
# the client is pointed at the server name instead.
hyperhiveDomain = config.services.hyperhive.domain;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# Same runtime→build-time bridge hive-forge and swarm-ci-runner cross:
# binds the hive trust bundle (which folds in the store's root) into the
# container and orders the container after `hive-tls-ca.service`. The
@ -1096,6 +1098,7 @@ in
systemd.services."container@hive-matrix" = lib.mkMerge [
{ after = [ "dnsmasq.service" ]; }
caTrust.containerOrdering
(rootfsCheck.warnOnly "hive-matrix")
];
};
}

View file

@ -0,0 +1,34 @@
# The pre-start check of a container's root, for one of hyperhive's own
# declarative containers.
#
# nixpkgs' start script runs as host root against the container's own root,
# which is persistent and writable by the container, and its `mkdir`, `chmod`,
# `touch` and `cp` follow links the container planted there. `hive-priv
# check-rootfs` looks at the paths the script touches (rules in
# `hive-priv/src/prestart.rs`). With `--warn-only` it logs each violation at
# warning level and lets the start go on. The line carries systemd's `-`
# prefix, so a crash or a failed exec of the check cannot fail the start either.
# The agent containers run it strict
# from their per-agent drop-in (hive-priv's `WriteResourceLimits`).
#
# Pure function, not a NixOS module. A container's module calls it with its
# own name and folds the result into its unit:
#
# rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# # systemd.services."container@${machine}" = lib.mkMerge [ (rootfsCheck.warnOnly machine) … ];
#
# The name list is the callers', never `config.containers`: that also holds
# containers the operator runs that hyperhive doesn't own, and those get
# nothing. nixpkgs renders each declarative container as its own
# `container@<name>` unit (`nixos-containers.nix`, the `mapAttrsToList` that
# builds `container@${name}`), so a setting on that unit reaches that
# container only.
{ config }:
{
# Attributes for `systemd.services."container@<name>"`.
warnOnly = name: {
serviceConfig.ExecStartPre = [
"-${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}"
];
};
}

View file

@ -114,6 +114,8 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The forwarder reaches the swarm collector and authelia by name over TLS; in
# self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix {
@ -226,6 +228,7 @@ in
# CA ordering keeps the trust bundle's bind source there first.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -183,6 +183,8 @@ let
refreshConsumer = import ./lib/refresh-consumer.nix { };
storeSeededSecret = import ./lib/store-seeded-secret.nix { };
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib gatewayCfg;
tlsCfg = deployCfg.hive-controller.tls;
@ -970,6 +972,7 @@ in
# not be up.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly cfg.machine)
(lib.mkIf publishes {
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -1689,6 +1689,8 @@ let
forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}";
autheliaCfg = hyperhiveCfg.swarm.authelia;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# Outbound TLS for that collector: it dials the swarm collector's gateway
# name and authelia's token endpoint, both leaves under the store's runtime
# pki root, which cannot be baked into a derivation. Same helper, same two
@ -4091,6 +4093,7 @@ in
# cannot start is a store that is down.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly cfg.machine)
{ after = [ "systemd-tmpfiles-resetup.service" ]; }
];

View file

@ -52,6 +52,8 @@ let
# containers' forwarders all the same.
forwarderTelemetryPort = 8892;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The daemon reaches authelia, the store and the queue by name over TLS, and
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
# CA.
@ -165,6 +167,7 @@ in
# the unit does not exist.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -43,6 +43,8 @@ let
# below compares the two, so both come from here.
forgeUrl = "https://${forgeCfg.domain}";
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The forge's gateway serves a leaf from the store's `pki` mount in
# self-signed mode.
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
@ -329,6 +331,7 @@ in
# minting unit in and waits for it, as ./swarm-builder.nix's does.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -54,6 +54,8 @@ let
# containers' forwarders all the same.
forwarderTelemetryPort = 8893;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
@ -146,6 +148,7 @@ in
# missing, so where this host mints the leaf the container waits for it.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{ after = [ "swarm-bao-pki.service" ]; }
];

View file

@ -24,6 +24,8 @@ let
baoCfg = hyperhiveCfg.swarm.bao;
baoDeploy = deployCfg.bao;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
caTrust = import ./lib/hive-ca-trust.nix {
inherit lib gatewayCfg;
tlsCfg = deployCfg.hive-controller.tls;
@ -686,6 +688,7 @@ in
# ./swarm-builder.nix's does.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly cfg.machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -33,6 +33,8 @@ let
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
gatewayCfg = config.services.hyperhive.gateway;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# The responder introspects authelia over https BY NAME. Its HTTP client is
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
@ -961,7 +963,10 @@ in
# Order the container after the host CA generator, so the bind source
# exists before nspawn sets the mount up. Without it a late CA fails the
# container start outright rather than degrading.
systemd.services."container@${machine}" = caTrust.containerOrdering;
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
];
# Deliver the responder's store identity into the container before it
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses

View file

@ -44,6 +44,8 @@ let
# containers' forwarders all the same.
forwarderTelemetryPort = 8902;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
@ -134,6 +136,7 @@ in
# missing, so where this host mints the leaf the container waits for it.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{ after = [ "swarm-bao-pki.service" ]; }
];

View file

@ -173,6 +173,8 @@ let
# that ever names the socket.
hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The swarm's authelia is reached by its gateway name, whose leaf is
# issued by the swarm services sub-CA — so this container needs the same
# runtime CA trust every other consumer of a swarm-service name needs.
@ -706,7 +708,10 @@ in
# The CA bind source is written at runtime by a host unit, so the
# container has to start after it — otherwise nspawn sets up a mount
# over a file that does not exist yet.
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly cfg.machine)
];
# A collector that is up but not listening on a socket the gateway names
# answers every producer with a 502 and fails no unit. This one fails when

View file

@ -57,6 +57,8 @@ let
# The standard OTLP/HTTP port, on the container's loopback.
forwarderOtlpPort = 4318;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The daemon reaches authelia, the store and the queue by name over TLS, and
# in self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
@ -172,6 +174,7 @@ in
# the unit does not exist.
systemd.services."container@${machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -99,6 +99,8 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
# The forwarder reaches the swarm collector and authelia by name over TLS; in
# self-signed mode their leaves chain to the hive CA.
caTrust = import ./lib/hive-ca-trust.nix {
@ -290,6 +292,7 @@ in
# ordering keeps the trust bundle's bind source there first.
systemd.services."container@${cfg.machine}" = lib.mkMerge [
caTrust.containerOrdering
(rootfsCheck.warnOnly cfg.machine)
{
after = [ "swarm-bao-pki.service" ];
wants = [ "swarm-bao-pki.service" ];

View file

@ -198,6 +198,7 @@ let
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
refreshConsumer = import ./lib/refresh-consumer.nix { };
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
in
{
# What the store IS from any hive's point of view, the name it answers on,
@ -728,6 +729,8 @@ in
systemd.timers.${forwarderFetchUnit} =
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
systemd.services."container@${cfg.machine}" = rootfsCheck.warnOnly cfg.machine;
containers.${cfg.machine} = {
autoStart = true;
ephemeral = false;

View file

@ -19,6 +19,7 @@ let
;
})
hive
hiveWith
hiveWithController
runGroup
forceServiceConfigs
@ -73,6 +74,13 @@ let
gateway.openFirewall = true;
};
# A container the operator runs that hyperhive doesn't own, next to one it
# does (the classifier's).
withForeignContainer = hiveWith [ { containers.operator-box.config = { }; } ] {
deploy.bao.enable = true;
deploy.review-classifier.enable = true;
};
# On the store's host, which mints the runner's store identity.
withCi = hive {
deploy.bao.enable = true;
@ -1497,6 +1505,105 @@ let
&& (sc.ProtectKernelLogs or false)
&& (sc.ProtectKernelModules or false);
}
{
# Hyperhive's own declarative containers each run the root check from
# their own unit, warn-only. Every container in these fixtures is one of
# hyperhive's, so the names are read off the fixture: a container added
# without the check breaks the list, which is the point. The unit's own
# pre-start script is not a `hive-priv` entry, so the count is the check
# alone.
name = "every hyperhive container runs the root check warn-only from its own unit, and no hyperhive container is left out";
ok =
let
checks =
m: name:
let
pre = m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ];
in
builtins.filter (lib.hasInfix "hive-priv") (
map (p: builtins.unsafeDiscardStringContext (toString p)) pre
);
carries =
m: name:
checks m name == [
(builtins.unsafeDiscardStringContext "-${m.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}")
];
names = lib.attrNames allLocal.containers;
in
names == [
"hive-forge"
"hive-matrix"
"swarm-attic"
"swarm-authelia"
"swarm-bao"
"swarm-builder"
"swarm-claude-login"
"swarm-grafana"
"swarm-nats"
"swarm-notify-router"
"swarm-otel"
"swarm-review-classifier"
"swarm-victorialogs"
"swarm-victoriametrics"
]
&& lib.all (carries allLocal) names
&& withCi.containers ? swarm-ci-runner
&& lib.all (carries withCi) (lib.attrNames withCi.containers)
# Control: a name with no unit finds nothing.
&& checks allLocal "swarm-nope" == [ ];
}
{
# Advisory means the start goes on whatever the check does, a crash or a
# missing binary included: systemd ignores the failure of a command
# prefixed `-`. The agents' strict line is the opposite and has no
# prefix; hive-priv renders it at runtime, and its own test
# (`the_rootfs_check_leads_the_service_section_…`) pins the bare
# `ExecStartPre=` there.
name = "every hyperhive container's root check is ignored on failure, not left to fail the start";
ok =
let
hivePrivLines =
m: name:
builtins.filter (lib.hasInfix "hive-priv") (
map (p: builtins.unsafeDiscardStringContext (toString p)) (
m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]
)
);
allLines = m: lib.concatMap (hivePrivLines m) (lib.attrNames m.containers);
ignored = lib.hasPrefix "-/";
in
allLines allLocal != [ ]
&& lib.length (allLines allLocal) == lib.length (lib.attrNames allLocal.containers)
&& lib.all ignored (allLines allLocal)
&& lib.all ignored (allLines withCi)
# Control: the test discriminates, a line without the prefix is not ignored.
&& !(ignored "/nix/store/x-hyperhive/bin/hive-priv check-rootfs --warn-only x");
}
{
# The operator's other containers are not hyperhive's to touch: a
# `containers.<name>` it declares gets no check, while the classifier
# beside it does. The positive arm is what makes the absence mean
# something: the same lookup finds the classifier's.
name = "a container hyperhive does not declare gets no root check";
ok =
let
checks =
name:
builtins.filter (lib.hasInfix "hive-priv") (
map (p: builtins.unsafeDiscardStringContext (toString p)) (
withForeignContainer.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]
)
);
in
lib.attrNames withForeignContainer.containers == [
"operator-box"
"swarm-bao"
"swarm-review-classifier"
]
&& lib.length (checks "swarm-review-classifier") == 1
&& lib.length (checks "swarm-bao") == 1
&& checks "operator-box" == [ ];
}
]
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
# at one fires and fetches nothing.

View file

@ -42,8 +42,8 @@ let
# nixpkgs. `read-only.nix` makes any host module that defines top-level
# `nixpkgs.config`/`overlays`/`hostPlatform` fail the fixture's eval instead
# of being ignored.
hive =
extra:
hiveWith =
modules: extra:
(nixosSystem {
modules = [
(pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix")
@ -70,9 +70,13 @@ let
swarm.hives.h1.domain = "h1.t.local";
} extra;
}
];
]
++ modules;
}).config;
# `hiveWith` with no extra modules: the host a fixture normally is.
hive = hiveWith [ ];
# `hive` with the hive-controller tier on (hive-c0re, the agent containers,
# `deploy.hive-controller.*`). Only a fixture whose claim reads that tier
# takes this; every other claim is about the swarm services and evaluates
@ -299,7 +303,7 @@ let
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
in
{
inherit hive;
inherit hive hiveWith;
inherit hiveWithController;
inherit agent;
inherit agentWith;