Watch
0
0
Fork
You've already forked hyperhive
0

Compare commits

..
Author SHA1 Message Date
atlas
86e80250bb remove the partial salvage dir when a salvage fails
- a failure between creating the dir and the rename removes that dir only
- give the ephemeral-rootfs vm test a 4 GiB disk so the space check passes
2026-10-28 23:01:04 +01:00
atlas
836b15a46e harden the ephemeral-root salvage: clone, check space, lock
- clone each file with FICLONE, copy the data only where that fails
- skip the salvage when the copy would leave too little free space
- hold a per-container flock and one copy at a time across containers
- run WriteNspawnFlags on the blocking pool
2026-10-28 23:01:04 +01:00
9 changed files with 979 additions and 88 deletions

1
Cargo.lock generated
View file

@ -1990,6 +1990,7 @@ dependencies = [
"anyhow",
"hive-priv-sock",
"libc",
"rustix",
"serde_json",
"tokio",
"tracing",

View file

@ -414,6 +414,30 @@ which is the only gcroots path bound into a container, so only root writes it.
Destroying the container removes the drop-in and then its root; the build stays
until the next collection. A drop-in written by an older hive-priv has no
`ExecStartPre=` and no root, and the container starts as before.
**Disk use.** The copy shares blocks with the old root where it can: hive-priv
clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
copies the data of a file the filesystem refuses to clone. Before it creates
anything, hive-priv sizes the root and compares it with the free space under
`work/`. A salvage whose files clone (tested with one file of the root) must
leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem
free. When the free space is short, hive-priv skips the salvage, deletes
nothing, logs `salvage skipped: <container> need <size> free <size>`, and the
container starts persistent; the next start tries again. hive-priv checks a file
that fails to clone against the 2 GiB reserve again before it copies the data. A
salvage that fails partway removes the directory it created, and leaves the root
and any earlier salvage as they were. After a salvage, hive-priv logs `salvage: <container> <size> reflinked=<n>/<m> files`.
**Concurrency.** hive-priv holds an `flock` on
`/run/hive-priv/salvage-<container>.lock` (the unit's `RuntimeDirectory`, which
no container can write) for the whole `WriteNspawnFlags`, so a second request for
the same container waits and then finds the finished template. Across
containers, one salvage copies at a time, so the free space each one checks
excludes any copy still in flight. The request runs on the blocking
thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease
already serialises the requests it sends; the lock covers hive-priv's own
guarantee.
The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`)
reads the guest's own `/var/log/journal` and pushes it over OTLP, so no
journal link to the host takes part; nspawn skips `--link-journal=try-guest`

View file

@ -11,6 +11,7 @@ workspace = true
anyhow.workspace = true
hive-priv-sock.workspace = true
libc.workspace = true
rustix.workspace = true
serde_json.workspace = true
tokio.workspace = true
tracing.workspace = true

View file

@ -61,6 +61,32 @@ pub(crate) fn handle_write_nspawn_flags(
Ok((String::new(), String::new()))
}
/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside
/// it copies files for as long as the root is big, and would hold a runtime
/// worker for that time.
///
/// # Errors
/// As [`handle_write_nspawn_flags`], or when the task fails to join.
pub(crate) async fn handle_write_nspawn_flags_blocking(
container: String,
binds: Vec<BindMount>,
isolation: NetworkIsolation,
load_credentials: Vec<CredentialMount>,
hive_facts: HiveFacts,
) -> Result<(String, String)> {
tokio::task::spawn_blocking(move || {
handle_write_nspawn_flags(
&container,
&binds,
&isolation,
&load_credentials,
&hive_facts,
)
})
.await
.context("write nspawn flags task")?
}
/// Validate a system-level container name (`h-` prefix for all agents,
/// the manager included) and return the logical agent name after it.
/// Anything else is refused, the swarm's forge and matrix containers

View file

@ -20,8 +20,9 @@
//! dispatch table. Each op family's handlers and validation live in their
//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
//! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
//! turns ephemeral), [`prestart`] (the check of a container's root that runs
//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls),
//! [`prestart`] (the check of a container's root that runs before each start,
//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
//! the swarm cache). Helpers more than one family uses are in [`shared`].
@ -33,6 +34,7 @@ mod files;
mod infra;
mod prestart;
mod rootfs;
mod salvage_io;
mod shared;
mod storage;
@ -383,18 +385,21 @@ async fn exec(
PrivRequest::ListContainers => container::container_run(&["list"]).await,
PrivRequest::WriteNspawnFlags {
ref container,
ref binds,
ref isolation,
ref load_credentials,
ref hive_facts,
} => container_config::handle_write_nspawn_flags(
container,
binds,
isolation,
load_credentials,
hive_facts,
),
} => {
container_config::handle_write_nspawn_flags_blocking(
container,
binds,
isolation,
load_credentials,
hive_facts,
)
.await
}
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,84 @@
//! What the root salvage ([`crate::rootfs`]) asks of the host: free space,
//! reflinks, and the locks that keep two salvages apart.
//!
//! The space and reflink calls sit behind [`Backend`] so a test can set the
//! free space and make a clone fail.
use std::fs::{File, OpenOptions};
use std::io;
use std::os::unix::fs::OpenOptionsExt as _;
use std::path::Path;
use std::sync::Mutex;
use anyhow::{Context as _, Result};
/// Where the per-container lock files live. A root-owned runtime directory
/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it.
pub(crate) const LOCK_DIR: &str = "/run/hive-priv";
/// Held by the one salvage that is copying data at any moment.
pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
/// Size and free space of a filesystem, in bytes. `free` is what an
/// unprivileged writer could still use.
#[derive(Debug, Clone, Copy)]
pub(crate) struct FsSpace {
pub(crate) free: u64,
pub(crate) total: u64,
}
/// The host calls the salvage makes.
pub(crate) trait Backend {
/// Space on the filesystem holding `path`.
fn space(&self, path: &Path) -> io::Result<FsSpace>;
/// Make `dst`, an empty file, share `src`'s data blocks. An error leaves
/// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on
/// different ones.
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>;
}
/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`.
pub(crate) struct Os;
impl Backend for Os {
fn space(&self, path: &Path) -> io::Result<FsSpace> {
let st = rustix::fs::statvfs(path)?;
Ok(FsSpace {
free: st.f_bavail.saturating_mul(st.f_frsize),
total: st.f_blocks.saturating_mul(st.f_frsize),
})
}
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
Ok(rustix::fs::ioctl_ficlone(dst, src)?)
}
}
/// An exclusive `flock(2)` on a container's lock file, released on drop.
pub(crate) struct ContainerLock {
_file: File,
}
impl ContainerLock {
/// Wait for the lock of `name` in `dir`, creating the file if needed.
///
/// # Errors
/// Returns an error when the directory or file cannot be created or the
/// lock call fails.
pub(crate) fn acquire(dir: &Path, name: &str) -> Result<Self> {
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
let path = dir.join(format!("salvage-{name}.lock"));
let file = OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW)
.open(&path)
.with_context(|| format!("open {}", path.display()))?;
file.lock()
.with_context(|| format!("lock {}", path.display()))?;
Ok(Self { _file: file })
}
}

View file

@ -146,6 +146,10 @@ in
];
# Writable HOME for nix's caches (see environment.HOME above).
StateDirectory = "hive-priv";
# /run/hive-priv: the per-container salvage lock files. Root-owned and
# outside every path a container can write.
RuntimeDirectory = "hive-priv";
RuntimeDirectoryMode = "0700";
# With ProtectSystem=strict the root filesystem is read-only inside
# hive-priv. When `nixos-container create/update` invokes nix, nix
# creates a temporary result symlink in its working directory. Without

View file

@ -78,6 +78,9 @@ pkgs.testers.runNixOSTest {
pkgs.hello
pkgs.tree
];
# The salvage skips when a copy would leave under 15% of the disk free, and
# this disk cannot clone files; the default 1 GiB leaves too little room.
virtualisation.diskSize = 4096;
# The agent bridge hive-priv names in the container's conf.
networking.bridges.${bridge}.interfaces = [ ];
networking.interfaces.${bridge}.ipv4.addresses = [