Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e0d2350902 | ||
|
|
24afc89ab3 | ||
|
|
c191975aab |
20 changed files with 444 additions and 40 deletions
|
|
@ -415,6 +415,17 @@ Destroying the container removes the drop-in and then its root; the build stays
|
|||
until the next collection. A drop-in written by an older hive-priv has no
|
||||
`ExecStartPre=` and no root, and the container starts as before.
|
||||
|
||||
**Hyperhive's platform containers: warn only.** The forge, the matrix server and
|
||||
the swarm services (15 containers, declared by their own modules) run the same
|
||||
check as `hive-priv check-rootfs --warn-only <name>`, from `ExecStartPre=` on
|
||||
their own `container@<name>` unit. It evaluates every rule, logs each violation
|
||||
as a warning, and exits 0, so the container starts either way. The line reads
|
||||
`rootfs check (warn-only): <name>: <message>` with journald's warning priority.
|
||||
Strict refusal needs the live roots of all 15 to pass first, and the warnings
|
||||
show which ones would not. A container the operator runs that hyperhive doesn't
|
||||
declare gets no check at all: the modules name their own containers, and
|
||||
nothing enumerates the host's `containers`.
|
||||
|
||||
**Disk use.** The copy shares blocks with the old root where it can: hive-priv
|
||||
clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
|
||||
copies the data of a file the filesystem refuses to clone. Before it creates
|
||||
|
|
|
|||
|
|
@ -3,12 +3,17 @@
|
|||
//! nixpkgs' `container@.service` start script runs as host root against the
|
||||
//! container's own root, which is persistent and writable by the container.
|
||||
//! `mkdir -p`, `chmod`, `touch` and `cp --remove-destination` there follow a
|
||||
//! link the container planted at any of the names the script touches. The
|
||||
//! per-container drop-in ([`crate::container_config`]) puts
|
||||
//! link the container planted at any of the names the script touches. An agent
|
||||
//! container's drop-in ([`crate::container_config`]) puts
|
||||
//! `hive-priv check-rootfs <container>` ahead of that script as an
|
||||
//! `ExecStartPre=`; a non-zero exit fails the start. The container is stopped
|
||||
//! while this runs, so nothing changes under the walk.
|
||||
//!
|
||||
//! Hyperhive's own declarative containers (the forge, the matrix server, the
|
||||
//! swarm services) run it as `check-rootfs --warn-only <container>` from their
|
||||
//! own unit (`nix/host-modules/lib/container-rootfs-check.nix`): each violation
|
||||
//! is logged at warning level and the start goes on.
|
||||
//!
|
||||
//! Every component is looked at with `symlink_metadata`: a link is reported,
|
||||
//! never followed. Nothing is deleted or repaired, so whatever is there stays
|
||||
//! for the operator.
|
||||
|
|
@ -27,11 +32,15 @@ use std::path::{Path, PathBuf};
|
|||
use anyhow::{Context as _, Result, bail};
|
||||
|
||||
use crate::container_config::{container_rootfs, validate_container_system_name};
|
||||
use crate::shared::partial_name;
|
||||
use crate::shared::{partial_name, validate_name_chars};
|
||||
|
||||
/// The sub-command the drop-in's `ExecStartPre=` names.
|
||||
pub(crate) const SUBCOMMAND: &str = "check-rootfs";
|
||||
|
||||
/// The flag that turns a violation into a warning: the check runs in full and
|
||||
/// exits 0.
|
||||
const WARN_ONLY: &str = "--warn-only";
|
||||
|
||||
/// The per-container nix profile directories on the host.
|
||||
const PROFILES_ROOT: &str = "/nix/var/nix/profiles/per-container";
|
||||
|
||||
|
|
@ -135,33 +144,84 @@ fn unpin_binary_in(roots: &Path, container: &str) -> Result<()> {
|
|||
/// Handle `hive-priv <subcommand> ...` when the first argument names one.
|
||||
/// `None` means it is not a sub-command and the daemon starts.
|
||||
///
|
||||
/// Prints one line to stderr and returns exit code 1 on a violation, so the
|
||||
/// reason lands in the journal next to the failed unit.
|
||||
/// `check-rootfs <container>` is strict: it prints one line to stderr and
|
||||
/// returns exit code 1 on a violation, so the reason lands in the journal next
|
||||
/// to the failed unit. `check-rootfs --warn-only <container>` prints every
|
||||
/// violation as a warning and returns 0.
|
||||
pub(crate) fn run(args: &[String]) -> Option<i32> {
|
||||
if args.first().map(String::as_str) != Some(SUBCOMMAND) {
|
||||
return None;
|
||||
}
|
||||
let result = match args {
|
||||
[_, container] => check_container(container),
|
||||
_ => Err(anyhow::anyhow!("usage: hive-priv {SUBCOMMAND} <container>")),
|
||||
};
|
||||
Some(match result {
|
||||
Ok(()) => 0,
|
||||
Err(e) => {
|
||||
eprintln!("hive-priv {SUBCOMMAND}: refusing to start: {e:#}");
|
||||
1
|
||||
let (container, warn_only) = match args {
|
||||
[_, container] => (container, false),
|
||||
[_, flag, container] if flag == WARN_ONLY => (container, true),
|
||||
_ => {
|
||||
eprintln!("usage: hive-priv {SUBCOMMAND} [{WARN_ONLY}] <container>");
|
||||
return Some(1);
|
||||
}
|
||||
})
|
||||
};
|
||||
let found = found_in_container(container, warn_only);
|
||||
let (code, lines) = report(container, &found, warn_only);
|
||||
for line in lines {
|
||||
eprintln!("{line}");
|
||||
}
|
||||
Some(code)
|
||||
}
|
||||
|
||||
/// [`check`] on the host paths of `container`.
|
||||
fn check_container(container: &str) -> Result<()> {
|
||||
validate_container_system_name(container)?;
|
||||
check(
|
||||
&container_rootfs(container),
|
||||
&Path::new(PROFILES_ROOT).join(container).join("system"),
|
||||
Path::new(STORE),
|
||||
)
|
||||
/// What a start would be refused for, on the host paths of `container`.
|
||||
///
|
||||
/// Strict mode stops at the first violation, as the start would. Warn-only
|
||||
/// lists them all, and counts a name it can't use as one more: it never fails
|
||||
/// the start over its own input.
|
||||
fn found_in_container(container: &str, warn_only: bool) -> Vec<String> {
|
||||
let paths = container_paths(container, warn_only);
|
||||
match (paths, warn_only) {
|
||||
(Ok((root, system)), true) => violations(&root, &system, Path::new(STORE)),
|
||||
(Ok((root, system)), false) => check(&root, &system, Path::new(STORE))
|
||||
.err()
|
||||
.map(|e| format!("{e:#}"))
|
||||
.into_iter()
|
||||
.collect(),
|
||||
(Err(e), _) => vec![format!("{e:#}")],
|
||||
}
|
||||
}
|
||||
|
||||
/// The container's root and its profile's `system` link on the host.
|
||||
///
|
||||
/// An agent container is named `h-<agent>`. A warn-only check also takes the
|
||||
/// platform containers' names, which are lowercase ascii, digits and hyphens.
|
||||
/// Either way the name holds no `/` and is never `.` or `..`.
|
||||
fn container_paths(container: &str, warn_only: bool) -> Result<(PathBuf, PathBuf)> {
|
||||
if warn_only {
|
||||
validate_name_chars(container)?;
|
||||
} else {
|
||||
validate_container_system_name(container)?;
|
||||
}
|
||||
Ok((
|
||||
container_rootfs(container),
|
||||
Path::new(PROFILES_ROOT).join(container).join("system"),
|
||||
))
|
||||
}
|
||||
|
||||
/// The exit code and the stderr lines for the violations `found`.
|
||||
///
|
||||
/// Warn-only lines carry the `<4>` prefix, which journald reads as the
|
||||
/// warning priority, so a log query can select them by severity.
|
||||
fn report(container: &str, found: &[String], warn_only: bool) -> (i32, Vec<String>) {
|
||||
if warn_only {
|
||||
let lines = found
|
||||
.iter()
|
||||
.map(|v| format!("<4>rootfs check (warn-only): {container}: {v}"))
|
||||
.collect();
|
||||
return (0, lines);
|
||||
}
|
||||
match found.first() {
|
||||
Some(v) => (
|
||||
1,
|
||||
vec![format!("hive-priv {SUBCOMMAND}: refusing to start: {v}")],
|
||||
),
|
||||
None => (0, Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check the paths the start script touches as root, returning an error that
|
||||
|
|
@ -173,14 +233,36 @@ fn check_container(container: &str) -> Result<()> {
|
|||
/// A path that does not exist passes: a first start has nothing yet, and the
|
||||
/// script creates it.
|
||||
fn check(root: &Path, system: &Path, store: &Path) -> Result<()> {
|
||||
match violations(root, system, store).into_iter().next() {
|
||||
None => Ok(()),
|
||||
Some(first) => bail!("{first}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every path under the rules of [`check`] that holds something it must not,
|
||||
/// in the order [`check`] looks at them, each message once.
|
||||
fn violations(root: &Path, system: &Path, store: &Path) -> Vec<String> {
|
||||
let mut found = Vec::new();
|
||||
for dir in DIRS {
|
||||
check_dir(root, dir)?;
|
||||
note(&mut found, check_dir(root, dir));
|
||||
}
|
||||
for file in REGULAR_ONLY {
|
||||
check_file(root, file, false)?;
|
||||
note(&mut found, check_file(root, file, false));
|
||||
}
|
||||
note(&mut found, check_file(root, OS_RELEASE, true));
|
||||
note(&mut found, check_system(system, store));
|
||||
found
|
||||
}
|
||||
|
||||
/// Add the message of an error to `found`, unless it is already there: a link
|
||||
/// at `usr` is the finding for `usr/bin` too.
|
||||
fn note(found: &mut Vec<String>, result: Result<()>) {
|
||||
if let Err(e) = result {
|
||||
let message = format!("{e:#}");
|
||||
if !found.contains(&message) {
|
||||
found.push(message);
|
||||
}
|
||||
}
|
||||
check_file(root, OS_RELEASE, true)?;
|
||||
check_system(system, store)
|
||||
}
|
||||
|
||||
/// `symlink_metadata`, with `None` for a path that does not exist. The one
|
||||
|
|
@ -285,7 +367,7 @@ mod tests {
|
|||
use std::os::unix::fs::symlink;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::{check, pin_binary_in, run, unpin_binary_in};
|
||||
use super::{check, container_paths, pin_binary_in, report, run, unpin_binary_in, violations};
|
||||
use crate::shared::test_support::{entries, scratch};
|
||||
|
||||
/// A scratch container: root, the profile link's path and a store with one
|
||||
|
|
@ -608,16 +690,133 @@ mod tests {
|
|||
assert!(msg.contains("left alone"), "{msg}");
|
||||
assert!(roots.join("h-iris/keep").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_check_rootfs_sub_command_is_taken() {
|
||||
let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::<Vec<_>>();
|
||||
assert_eq!(run(&[]), None);
|
||||
assert_eq!(run(&["serve".to_string()]), None);
|
||||
// Wrong arity and a name that is not an agent container are refused,
|
||||
// whatever is on disk.
|
||||
assert_eq!(run(&["check-rootfs".to_string()]), Some(1));
|
||||
assert_eq!(run(&args(&["serve"])), None);
|
||||
// Wrong arity, an unknown flag and a name that is not an agent
|
||||
// container are refused, whatever is on disk.
|
||||
assert_eq!(run(&args(&["check-rootfs"])), Some(1));
|
||||
assert_eq!(run(&args(&["check-rootfs", "not-hive"])), Some(1));
|
||||
assert_eq!(run(&args(&["check-rootfs", "--strict", "h-iris"])), Some(1));
|
||||
assert_eq!(run(&args(&["check-rootfs", "--warn-only"])), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_warn_only_run_exits_zero_even_for_a_name_it_cannot_use() {
|
||||
let args = |a: &[&str]| a.iter().map(|s| (*s).to_owned()).collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
run(&["check-rootfs".to_string(), "not-hive".to_string()]),
|
||||
Some(1)
|
||||
run(&args(&["check-rootfs", "--warn-only", "no-such-container"])),
|
||||
Some(0)
|
||||
);
|
||||
assert_eq!(
|
||||
run(&args(&["check-rootfs", "--warn-only", "../etc"])),
|
||||
Some(0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_takes_agent_names_and_warn_only_takes_plain_ones() {
|
||||
assert!(container_paths("h-iris", false).is_ok());
|
||||
assert!(container_paths("hive-forge", false).is_err());
|
||||
for name in ["hive-forge", "swarm-review-classifier", "h-iris"] {
|
||||
let (root, system) = container_paths(name, true).unwrap();
|
||||
assert_eq!(
|
||||
root,
|
||||
PathBuf::from(format!("/var/lib/nixos-containers/{name}"))
|
||||
);
|
||||
assert_eq!(
|
||||
system,
|
||||
PathBuf::from(format!("/nix/var/nix/profiles/per-container/{name}/system"))
|
||||
);
|
||||
}
|
||||
for name in [
|
||||
"",
|
||||
".",
|
||||
"..",
|
||||
"../h-iris",
|
||||
"a/b",
|
||||
"/etc",
|
||||
"H-iris",
|
||||
"a.b",
|
||||
"a b",
|
||||
] {
|
||||
assert!(
|
||||
container_paths(name, true).is_err(),
|
||||
"{name:?} was accepted"
|
||||
);
|
||||
assert!(
|
||||
container_paths(name, false).is_err(),
|
||||
"{name:?} was accepted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A clean root with a link at `root` and another at `etc/resolv.conf`.
|
||||
fn two_violations() -> Fixture {
|
||||
let f = Fixture::clean();
|
||||
std::fs::remove_dir_all(f.root.join("root")).unwrap();
|
||||
symlink(victim(), f.root.join("root")).unwrap();
|
||||
std::fs::remove_file(f.root.join("etc/resolv.conf")).unwrap();
|
||||
symlink("/etc/hosts", f.root.join("etc/resolv.conf")).unwrap();
|
||||
f
|
||||
}
|
||||
|
||||
fn found(f: &Fixture) -> Vec<String> {
|
||||
violations(&f.root, &f.system, &f.store)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warn_only_logs_every_violation_at_warning_level_and_exits_zero() {
|
||||
let f = two_violations();
|
||||
let found = found(&f);
|
||||
assert_eq!(found.len(), 2, "{found:?}");
|
||||
let (code, lines) = report("hive-forge", &found, true);
|
||||
assert_eq!(code, 0);
|
||||
assert_eq!(lines.len(), 2, "{lines:?}");
|
||||
for line in &lines {
|
||||
assert!(
|
||||
line.starts_with("<4>rootfs check (warn-only): hive-forge: "),
|
||||
"{line}"
|
||||
);
|
||||
}
|
||||
assert!(lines[0].contains("root is a symlink"), "{lines:?}");
|
||||
assert!(lines[1].contains("resolv.conf is a symlink"), "{lines:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_refuses_with_the_first_violation_only() {
|
||||
let f = two_violations();
|
||||
let found = found(&f);
|
||||
let (code, lines) = report("h-iris", &found, false);
|
||||
assert_eq!(code, 1);
|
||||
assert_eq!(lines.len(), 1, "{lines:?}");
|
||||
assert!(
|
||||
lines[0].starts_with("hive-priv check-rootfs: refusing to start: "),
|
||||
"{lines:?}"
|
||||
);
|
||||
assert!(lines[0].contains("root is a symlink"), "{lines:?}");
|
||||
// The rules are the same: the strict refusal is `check`'s first error.
|
||||
assert_eq!(f.refused(), found[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clean_root_prints_nothing_in_either_mode() {
|
||||
let f = Fixture::clean();
|
||||
assert!(found(&f).is_empty());
|
||||
assert_eq!(report("hive-forge", &[], true), (0, Vec::new()));
|
||||
assert_eq!(report("h-iris", &[], false), (0, Vec::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_link_is_reported_once_though_two_paths_run_through_it() {
|
||||
let f = Fixture::clean();
|
||||
std::fs::remove_dir_all(f.root.join("usr")).unwrap();
|
||||
symlink(victim(), f.root.join("usr")).unwrap();
|
||||
let found = found(&f);
|
||||
assert_eq!(found.len(), 1, "{found:?}");
|
||||
assert!(found[0].contains("usr is a symlink"), "{found:?}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,8 @@ let
|
|||
deployCfg.bao.forgePublisherClientCertFile != null
|
||||
&& deployCfg.bao.forgePublisherClientKeyFile != null;
|
||||
|
||||
rootfsCheck = import ../lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
||||
# The container's journal forwarder pushes to its own receiver on the swarm
|
||||
|
|
@ -471,6 +473,7 @@ in
|
|||
# ../swarm-builder.nix's does.
|
||||
systemd.services."container@hive-forge" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly "hive-forge")
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ let
|
|||
# the client is pointed at the server name instead.
|
||||
hyperhiveDomain = config.services.hyperhive.domain;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# Same runtime→build-time bridge hive-forge and swarm-ci-runner cross:
|
||||
# binds the hive trust bundle (which folds in the store's root) into the
|
||||
# container and orders the container after `hive-tls-ca.service`. The
|
||||
|
|
@ -1096,6 +1098,7 @@ in
|
|||
systemd.services."container@hive-matrix" = lib.mkMerge [
|
||||
{ after = [ "dnsmasq.service" ]; }
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly "hive-matrix")
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
|
|||
34
nix/host-modules/lib/container-rootfs-check.nix
Normal file
34
nix/host-modules/lib/container-rootfs-check.nix
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
# The pre-start check of a container's root, for one of hyperhive's own
|
||||
# declarative containers.
|
||||
#
|
||||
# nixpkgs' start script runs as host root against the container's own root,
|
||||
# which is persistent and writable by the container, and its `mkdir`, `chmod`,
|
||||
# `touch` and `cp` follow links the container planted there. `hive-priv
|
||||
# check-rootfs` looks at the paths the script touches (rules in
|
||||
# `hive-priv/src/prestart.rs`). With `--warn-only` it logs each violation at
|
||||
# warning level and lets the start go on. The line carries systemd's `-`
|
||||
# prefix, so a crash or a failed exec of the check cannot fail the start either.
|
||||
# The agent containers run it strict
|
||||
# from their per-agent drop-in (hive-priv's `WriteResourceLimits`).
|
||||
#
|
||||
# Pure function, not a NixOS module. A container's module calls it with its
|
||||
# own name and folds the result into its unit:
|
||||
#
|
||||
# rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
# # systemd.services."container@${machine}" = lib.mkMerge [ (rootfsCheck.warnOnly machine) … ];
|
||||
#
|
||||
# The name list is the callers', never `config.containers`: that also holds
|
||||
# containers the operator runs that hyperhive doesn't own, and those get
|
||||
# nothing. nixpkgs renders each declarative container as its own
|
||||
# `container@<name>` unit (`nixos-containers.nix`, the `mapAttrsToList` that
|
||||
# builds `container@${name}`), so a setting on that unit reaches that
|
||||
# container only.
|
||||
{ config }:
|
||||
{
|
||||
# Attributes for `systemd.services."container@<name>"`.
|
||||
warnOnly = name: {
|
||||
serviceConfig.ExecStartPre = [
|
||||
"-${config.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}"
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
@ -114,6 +114,8 @@ let
|
|||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
|
|
@ -226,6 +228,7 @@ in
|
|||
# CA ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -183,6 +183,8 @@ let
|
|||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
storeSeededSecret = import ./lib/store-seeded-secret.nix { };
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
|
|
@ -970,6 +972,7 @@ in
|
|||
# not be up.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
(lib.mkIf publishes {
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -1689,6 +1689,8 @@ let
|
|||
forwarderAuthName = "oauth2client/${swarmOtelCfg.storeProducerName}";
|
||||
autheliaCfg = hyperhiveCfg.swarm.authelia;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# Outbound TLS for that collector: it dials the swarm collector's gateway
|
||||
# name and authelia's token endpoint, both leaves under the store's runtime
|
||||
# pki root, which cannot be baked into a derivation. Same helper, same two
|
||||
|
|
@ -4091,6 +4093,7 @@ in
|
|||
# cannot start is a store that is down.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{ after = [ "systemd-tmpfiles-resetup.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8892;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# `git` reaches the forge; in self-signed mode their leaves chain to the hive
|
||||
# CA.
|
||||
|
|
@ -165,6 +167,7 @@ in
|
|||
# the unit does not exist.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ let
|
|||
# below compares the two, so both come from here.
|
||||
forgeUrl = "https://${forgeCfg.domain}";
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forge's gateway serves a leaf from the store's `pki` mount in
|
||||
# self-signed mode.
|
||||
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
||||
|
|
@ -329,6 +331,7 @@ in
|
|||
# minting unit in and waits for it, as ./swarm-builder.nix's does.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8893;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -146,6 +148,7 @@ in
|
|||
# missing, so where this host mints the leaf the container waits for it.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{ after = [ "swarm-bao-pki.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ let
|
|||
baoCfg = hyperhiveCfg.swarm.bao;
|
||||
baoDeploy = deployCfg.bao;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
inherit lib gatewayCfg;
|
||||
tlsCfg = deployCfg.hive-controller.tls;
|
||||
|
|
@ -686,6 +688,7 @@ in
|
|||
# ./swarm-builder.nix's does.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ let
|
|||
|
||||
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
||||
gatewayCfg = config.services.hyperhive.gateway;
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
# The responder introspects authelia over https BY NAME. Its HTTP client is
|
||||
# reqwest/rustls, and `rustls-platform-verifier` resolves roots through
|
||||
|
|
@ -961,7 +963,10 @@ in
|
|||
# Order the container after the host CA generator, so the bind source
|
||||
# exists before nspawn sets the mount up. Without it a late CA fails the
|
||||
# container start outright rather than degrading.
|
||||
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
];
|
||||
|
||||
# Deliver the responder's store identity into the container before it
|
||||
# starts. A copy rather than a `bindMounts` entry: nixos-container refuses
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ let
|
|||
# containers' forwarders all the same.
|
||||
forwarderTelemetryPort = 8902;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -134,6 +136,7 @@ in
|
|||
# missing, so where this host mints the leaf the container waits for it.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{ after = [ "swarm-bao-pki.service" ]; }
|
||||
];
|
||||
|
||||
|
|
|
|||
|
|
@ -173,6 +173,8 @@ let
|
|||
# that ever names the socket.
|
||||
hiveSockets = lib.genAttrs (lib.attrNames hyperhiveCfg.swarm.hives) socketOf;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The swarm's authelia is reached by its gateway name, whose leaf is
|
||||
# issued by the swarm services sub-CA — so this container needs the same
|
||||
# runtime CA trust every other consumer of a swarm-service name needs.
|
||||
|
|
@ -706,7 +708,10 @@ in
|
|||
# The CA bind source is written at runtime by a host unit, so the
|
||||
# container has to start after it — otherwise nspawn sets up a mount
|
||||
# over a file that does not exist yet.
|
||||
systemd.services."container@${cfg.machine}" = caTrust.containerOrdering;
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
];
|
||||
|
||||
# A collector that is up but not listening on a socket the gateway names
|
||||
# answers every producer with a 502 and fails no unit. This one fails when
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ let
|
|||
# The standard OTLP/HTTP port, on the container's loopback.
|
||||
forwarderOtlpPort = 4318;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The daemon reaches authelia, the store and the queue by name over TLS, and
|
||||
# in self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
||||
|
|
@ -172,6 +174,7 @@ in
|
|||
# the unit does not exist.
|
||||
systemd.services."container@${machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -99,6 +99,8 @@ let
|
|||
# one word.
|
||||
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
|
||||
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
|
||||
# The forwarder reaches the swarm collector and authelia by name over TLS; in
|
||||
# self-signed mode their leaves chain to the hive CA.
|
||||
caTrust = import ./lib/hive-ca-trust.nix {
|
||||
|
|
@ -290,6 +292,7 @@ in
|
|||
# ordering keeps the trust bundle's bind source there first.
|
||||
systemd.services."container@${cfg.machine}" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
(rootfsCheck.warnOnly cfg.machine)
|
||||
{
|
||||
after = [ "swarm-bao-pki.service" ];
|
||||
wants = [ "swarm-bao-pki.service" ];
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ let
|
|||
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
||||
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
||||
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
||||
rootfsCheck = import ./lib/container-rootfs-check.nix { inherit config; };
|
||||
in
|
||||
{
|
||||
# What the store IS from any hive's point of view, the name it answers on,
|
||||
|
|
@ -728,6 +729,8 @@ in
|
|||
systemd.timers.${forwarderFetchUnit} =
|
||||
storeRefreshTimer "re-read the metrics store forwarder's OIDC client secret from the swarm secret store";
|
||||
|
||||
systemd.services."container@${cfg.machine}" = rootfsCheck.warnOnly cfg.machine;
|
||||
|
||||
containers.${cfg.machine} = {
|
||||
autoStart = true;
|
||||
ephemeral = false;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ let
|
|||
;
|
||||
})
|
||||
hive
|
||||
hiveWith
|
||||
hiveWithController
|
||||
runGroup
|
||||
forceServiceConfigs
|
||||
|
|
@ -73,6 +74,13 @@ let
|
|||
gateway.openFirewall = true;
|
||||
};
|
||||
|
||||
# A container the operator runs that hyperhive doesn't own, next to one it
|
||||
# does (the classifier's).
|
||||
withForeignContainer = hiveWith [ { containers.operator-box.config = { }; } ] {
|
||||
deploy.bao.enable = true;
|
||||
deploy.review-classifier.enable = true;
|
||||
};
|
||||
|
||||
# On the store's host, which mints the runner's store identity.
|
||||
withCi = hive {
|
||||
deploy.bao.enable = true;
|
||||
|
|
@ -1497,6 +1505,105 @@ let
|
|||
&& (sc.ProtectKernelLogs or false)
|
||||
&& (sc.ProtectKernelModules or false);
|
||||
}
|
||||
{
|
||||
# Hyperhive's own declarative containers each run the root check from
|
||||
# their own unit, warn-only. Every container in these fixtures is one of
|
||||
# hyperhive's, so the names are read off the fixture: a container added
|
||||
# without the check breaks the list, which is the point. The unit's own
|
||||
# pre-start script is not a `hive-priv` entry, so the count is the check
|
||||
# alone.
|
||||
name = "every hyperhive container runs the root check warn-only from its own unit, and no hyperhive container is left out";
|
||||
ok =
|
||||
let
|
||||
checks =
|
||||
m: name:
|
||||
let
|
||||
pre = m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ];
|
||||
in
|
||||
builtins.filter (lib.hasInfix "hive-priv") (
|
||||
map (p: builtins.unsafeDiscardStringContext (toString p)) pre
|
||||
);
|
||||
carries =
|
||||
m: name:
|
||||
checks m name == [
|
||||
(builtins.unsafeDiscardStringContext "-${m.services.hyperhive.c0re.package}/bin/hive-priv check-rootfs --warn-only ${name}")
|
||||
];
|
||||
names = lib.attrNames allLocal.containers;
|
||||
in
|
||||
names == [
|
||||
"hive-forge"
|
||||
"hive-matrix"
|
||||
"swarm-attic"
|
||||
"swarm-authelia"
|
||||
"swarm-bao"
|
||||
"swarm-builder"
|
||||
"swarm-claude-login"
|
||||
"swarm-grafana"
|
||||
"swarm-nats"
|
||||
"swarm-notify-router"
|
||||
"swarm-otel"
|
||||
"swarm-review-classifier"
|
||||
"swarm-victorialogs"
|
||||
"swarm-victoriametrics"
|
||||
]
|
||||
&& lib.all (carries allLocal) names
|
||||
&& withCi.containers ? swarm-ci-runner
|
||||
&& lib.all (carries withCi) (lib.attrNames withCi.containers)
|
||||
# Control: a name with no unit finds nothing.
|
||||
&& checks allLocal "swarm-nope" == [ ];
|
||||
}
|
||||
{
|
||||
# Advisory means the start goes on whatever the check does, a crash or a
|
||||
# missing binary included: systemd ignores the failure of a command
|
||||
# prefixed `-`. The agents' strict line is the opposite and has no
|
||||
# prefix; hive-priv renders it at runtime, and its own test
|
||||
# (`the_rootfs_check_leads_the_service_section_…`) pins the bare
|
||||
# `ExecStartPre=` there.
|
||||
name = "every hyperhive container's root check is ignored on failure, not left to fail the start";
|
||||
ok =
|
||||
let
|
||||
hivePrivLines =
|
||||
m: name:
|
||||
builtins.filter (lib.hasInfix "hive-priv") (
|
||||
map (p: builtins.unsafeDiscardStringContext (toString p)) (
|
||||
m.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]
|
||||
)
|
||||
);
|
||||
allLines = m: lib.concatMap (hivePrivLines m) (lib.attrNames m.containers);
|
||||
ignored = lib.hasPrefix "-/";
|
||||
in
|
||||
allLines allLocal != [ ]
|
||||
&& lib.length (allLines allLocal) == lib.length (lib.attrNames allLocal.containers)
|
||||
&& lib.all ignored (allLines allLocal)
|
||||
&& lib.all ignored (allLines withCi)
|
||||
# Control: the test discriminates, a line without the prefix is not ignored.
|
||||
&& !(ignored "/nix/store/x-hyperhive/bin/hive-priv check-rootfs --warn-only x");
|
||||
}
|
||||
{
|
||||
# The operator's other containers are not hyperhive's to touch: a
|
||||
# `containers.<name>` it declares gets no check, while the classifier
|
||||
# beside it does. The positive arm is what makes the absence mean
|
||||
# something: the same lookup finds the classifier's.
|
||||
name = "a container hyperhive does not declare gets no root check";
|
||||
ok =
|
||||
let
|
||||
checks =
|
||||
name:
|
||||
builtins.filter (lib.hasInfix "hive-priv") (
|
||||
map (p: builtins.unsafeDiscardStringContext (toString p)) (
|
||||
withForeignContainer.systemd.services."container@${name}".serviceConfig.ExecStartPre or [ ]
|
||||
)
|
||||
);
|
||||
in
|
||||
lib.attrNames withForeignContainer.containers == [
|
||||
"operator-box"
|
||||
"swarm-bao"
|
||||
"swarm-review-classifier"
|
||||
]
|
||||
&& lib.length (checks "swarm-review-classifier") == 1
|
||||
&& lib.length (checks "swarm-bao") == 1
|
||||
&& checks "operator-box" == [ ];
|
||||
}
|
||||
]
|
||||
# Starting a unit that stays `active (exited)` is a no-op, so a timer aimed
|
||||
# at one fires and fetches nothing.
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ let
|
|||
# nixpkgs. `read-only.nix` makes any host module that defines top-level
|
||||
# `nixpkgs.config`/`overlays`/`hostPlatform` fail the fixture's eval instead
|
||||
# of being ignored.
|
||||
hive =
|
||||
extra:
|
||||
hiveWith =
|
||||
modules: extra:
|
||||
(nixosSystem {
|
||||
modules = [
|
||||
(pkgs.path + "/nixos/modules/misc/nixpkgs/read-only.nix")
|
||||
|
|
@ -70,9 +70,13 @@ let
|
|||
swarm.hives.h1.domain = "h1.t.local";
|
||||
} extra;
|
||||
}
|
||||
];
|
||||
]
|
||||
++ modules;
|
||||
}).config;
|
||||
|
||||
# `hiveWith` with no extra modules: the host a fixture normally is.
|
||||
hive = hiveWith [ ];
|
||||
|
||||
# `hive` with the hive-controller tier on (hive-c0re, the agent containers,
|
||||
# `deploy.hive-controller.*`). Only a fixture whose claim reads that tier
|
||||
# takes this; every other claim is about the swarm services and evaluates
|
||||
|
|
@ -299,7 +303,7 @@ let
|
|||
builtins.foldl' (acc: v: builtins.seq v acc) true vals;
|
||||
in
|
||||
{
|
||||
inherit hive;
|
||||
inherit hive hiveWith;
|
||||
inherit hiveWithController;
|
||||
inherit agent;
|
||||
inherit agentWith;
|
||||
|
|
|
|||
Loading…
Reference in a new issue