Compare commits
9 changed files with 88 additions and 979 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1990,7 +1990,6 @@ dependencies = [
|
|||
"anyhow",
|
||||
"hive-priv-sock",
|
||||
"libc",
|
||||
"rustix",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
|
|
|
|||
|
|
@ -414,30 +414,6 @@ which is the only gcroots path bound into a container, so only root writes it.
|
|||
Destroying the container removes the drop-in and then its root; the build stays
|
||||
until the next collection. A drop-in written by an older hive-priv has no
|
||||
`ExecStartPre=` and no root, and the container starts as before.
|
||||
|
||||
**Disk use.** The copy shares blocks with the old root where it can: hive-priv
|
||||
clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
|
||||
copies the data of a file the filesystem refuses to clone. Before it creates
|
||||
anything, hive-priv sizes the root and compares it with the free space under
|
||||
`work/`. A salvage whose files clone (tested with one file of the root) must
|
||||
leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem
|
||||
free. When the free space is short, hive-priv skips the salvage, deletes
|
||||
nothing, logs `salvage skipped: <container> need <size> free <size>`, and the
|
||||
container starts persistent; the next start tries again. hive-priv checks a file
|
||||
that fails to clone against the 2 GiB reserve again before it copies the data. A
|
||||
salvage that fails partway removes the directory it created, and leaves the root
|
||||
and any earlier salvage as they were. After a salvage, hive-priv logs `salvage: <container> <size> reflinked=<n>/<m> files`.
|
||||
|
||||
**Concurrency.** hive-priv holds an `flock` on
|
||||
`/run/hive-priv/salvage-<container>.lock` (the unit's `RuntimeDirectory`, which
|
||||
no container can write) for the whole `WriteNspawnFlags`, so a second request for
|
||||
the same container waits and then finds the finished template. Across
|
||||
containers, one salvage copies at a time, so the free space each one checks
|
||||
excludes any copy still in flight. The request runs on the blocking
|
||||
thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease
|
||||
already serialises the requests it sends; the lock covers hive-priv's own
|
||||
guarantee.
|
||||
|
||||
The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`)
|
||||
reads the guest's own `/var/log/journal` and pushes it over OTLP, so no
|
||||
journal link to the host takes part; nspawn skips `--link-journal=try-guest`
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ workspace = true
|
|||
anyhow.workspace = true
|
||||
hive-priv-sock.workspace = true
|
||||
libc.workspace = true
|
||||
rustix.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
|
|
|
|||
|
|
@ -61,32 +61,6 @@ pub(crate) fn handle_write_nspawn_flags(
|
|||
Ok((String::new(), String::new()))
|
||||
}
|
||||
|
||||
/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside
|
||||
/// it copies files for as long as the root is big, and would hold a runtime
|
||||
/// worker for that time.
|
||||
///
|
||||
/// # Errors
|
||||
/// As [`handle_write_nspawn_flags`], or when the task fails to join.
|
||||
pub(crate) async fn handle_write_nspawn_flags_blocking(
|
||||
container: String,
|
||||
binds: Vec<BindMount>,
|
||||
isolation: NetworkIsolation,
|
||||
load_credentials: Vec<CredentialMount>,
|
||||
hive_facts: HiveFacts,
|
||||
) -> Result<(String, String)> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
handle_write_nspawn_flags(
|
||||
&container,
|
||||
&binds,
|
||||
&isolation,
|
||||
&load_credentials,
|
||||
&hive_facts,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.context("write nspawn flags task")?
|
||||
}
|
||||
|
||||
/// Validate a system-level container name (`h-` prefix for all agents,
|
||||
/// the manager included) and return the logical agent name after it.
|
||||
/// Anything else is refused, the swarm's forge and matrix containers
|
||||
|
|
|
|||
|
|
@ -20,9 +20,8 @@
|
|||
//! dispatch table. Each op family's handlers and validation live in their
|
||||
//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
|
||||
//! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
|
||||
//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls),
|
||||
//! [`prestart`] (the check of a container's root that runs before each start,
|
||||
//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
|
||||
//! turns ephemeral), [`prestart`] (the check of a container's root that runs
|
||||
//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
|
||||
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
|
||||
//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
|
||||
//! the swarm cache). Helpers more than one family uses are in [`shared`].
|
||||
|
|
@ -34,7 +33,6 @@ mod files;
|
|||
mod infra;
|
||||
mod prestart;
|
||||
mod rootfs;
|
||||
mod salvage_io;
|
||||
mod shared;
|
||||
mod storage;
|
||||
|
||||
|
|
@ -385,21 +383,18 @@ async fn exec(
|
|||
PrivRequest::ListContainers => container::container_run(&["list"]).await,
|
||||
|
||||
PrivRequest::WriteNspawnFlags {
|
||||
ref container,
|
||||
ref binds,
|
||||
ref isolation,
|
||||
ref load_credentials,
|
||||
ref hive_facts,
|
||||
} => container_config::handle_write_nspawn_flags(
|
||||
container,
|
||||
binds,
|
||||
isolation,
|
||||
load_credentials,
|
||||
hive_facts,
|
||||
} => {
|
||||
container_config::handle_write_nspawn_flags_blocking(
|
||||
container,
|
||||
binds,
|
||||
isolation,
|
||||
load_credentials,
|
||||
hive_facts,
|
||||
)
|
||||
.await
|
||||
}
|
||||
),
|
||||
|
||||
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,84 +0,0 @@
|
|||
//! What the root salvage ([`crate::rootfs`]) asks of the host: free space,
|
||||
//! reflinks, and the locks that keep two salvages apart.
|
||||
//!
|
||||
//! The space and reflink calls sit behind [`Backend`] so a test can set the
|
||||
//! free space and make a clone fail.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::os::unix::fs::OpenOptionsExt as _;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
|
||||
/// Where the per-container lock files live. A root-owned runtime directory
|
||||
/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it.
|
||||
pub(crate) const LOCK_DIR: &str = "/run/hive-priv";
|
||||
|
||||
/// Held by the one salvage that is copying data at any moment.
|
||||
pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Size and free space of a filesystem, in bytes. `free` is what an
|
||||
/// unprivileged writer could still use.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct FsSpace {
|
||||
pub(crate) free: u64,
|
||||
pub(crate) total: u64,
|
||||
}
|
||||
|
||||
/// The host calls the salvage makes.
|
||||
pub(crate) trait Backend {
|
||||
/// Space on the filesystem holding `path`.
|
||||
fn space(&self, path: &Path) -> io::Result<FsSpace>;
|
||||
|
||||
/// Make `dst`, an empty file, share `src`'s data blocks. An error leaves
|
||||
/// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on
|
||||
/// different ones.
|
||||
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>;
|
||||
}
|
||||
|
||||
/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`.
|
||||
pub(crate) struct Os;
|
||||
|
||||
impl Backend for Os {
|
||||
fn space(&self, path: &Path) -> io::Result<FsSpace> {
|
||||
let st = rustix::fs::statvfs(path)?;
|
||||
Ok(FsSpace {
|
||||
free: st.f_bavail.saturating_mul(st.f_frsize),
|
||||
total: st.f_blocks.saturating_mul(st.f_frsize),
|
||||
})
|
||||
}
|
||||
|
||||
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
|
||||
Ok(rustix::fs::ioctl_ficlone(dst, src)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// An exclusive `flock(2)` on a container's lock file, released on drop.
|
||||
pub(crate) struct ContainerLock {
|
||||
_file: File,
|
||||
}
|
||||
|
||||
impl ContainerLock {
|
||||
/// Wait for the lock of `name` in `dir`, creating the file if needed.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when the directory or file cannot be created or the
|
||||
/// lock call fails.
|
||||
pub(crate) fn acquire(dir: &Path, name: &str) -> Result<Self> {
|
||||
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
|
||||
let path = dir.join(format!("salvage-{name}.lock"));
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.mode(0o600)
|
||||
.custom_flags(libc::O_NOFOLLOW)
|
||||
.open(&path)
|
||||
.with_context(|| format!("open {}", path.display()))?;
|
||||
file.lock()
|
||||
.with_context(|| format!("lock {}", path.display()))?;
|
||||
Ok(Self { _file: file })
|
||||
}
|
||||
}
|
||||
|
|
@ -146,10 +146,6 @@ in
|
|||
];
|
||||
# Writable HOME for nix's caches (see environment.HOME above).
|
||||
StateDirectory = "hive-priv";
|
||||
# /run/hive-priv: the per-container salvage lock files. Root-owned and
|
||||
# outside every path a container can write.
|
||||
RuntimeDirectory = "hive-priv";
|
||||
RuntimeDirectoryMode = "0700";
|
||||
# With ProtectSystem=strict the root filesystem is read-only inside
|
||||
# hive-priv. When `nixos-container create/update` invokes nix, nix
|
||||
# creates a temporary result symlink in its working directory. Without
|
||||
|
|
|
|||
|
|
@ -78,9 +78,6 @@ pkgs.testers.runNixOSTest {
|
|||
pkgs.hello
|
||||
pkgs.tree
|
||||
];
|
||||
# The salvage skips when a copy would leave under 15% of the disk free, and
|
||||
# this disk cannot clone files; the default 1 GiB leaves too little room.
|
||||
virtualisation.diskSize = 4096;
|
||||
# The agent bridge hive-priv names in the container's conf.
|
||||
networking.bridges.${bridge}.interfaces = [ ];
|
||||
networking.interfaces.${bridge}.ipv4.addresses = [
|
||||
|
|
|
|||
Loading…
Reference in a new issue