Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
86e80250bb | ||
|
|
836b15a46e |
9 changed files with 979 additions and 88 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1990,6 +1990,7 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"hive-priv-sock",
|
"hive-priv-sock",
|
||||||
"libc",
|
"libc",
|
||||||
|
"rustix",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
|
||||||
|
|
@ -414,6 +414,30 @@ which is the only gcroots path bound into a container, so only root writes it.
|
||||||
Destroying the container removes the drop-in and then its root; the build stays
|
Destroying the container removes the drop-in and then its root; the build stays
|
||||||
until the next collection. A drop-in written by an older hive-priv has no
|
until the next collection. A drop-in written by an older hive-priv has no
|
||||||
`ExecStartPre=` and no root, and the container starts as before.
|
`ExecStartPre=` and no root, and the container starts as before.
|
||||||
|
|
||||||
|
**Disk use.** The copy shares blocks with the old root where it can: hive-priv
|
||||||
|
clones each file (`FICLONE`), which on btrfs or xfs allocates metadata only, and
|
||||||
|
copies the data of a file the filesystem refuses to clone. Before it creates
|
||||||
|
anything, hive-priv sizes the root and compares it with the free space under
|
||||||
|
`work/`. A salvage whose files clone (tested with one file of the root) must
|
||||||
|
leave 2 GiB free. One that has to copy data must leave 15 % of the filesystem
|
||||||
|
free. When the free space is short, hive-priv skips the salvage, deletes
|
||||||
|
nothing, logs `salvage skipped: <container> need <size> free <size>`, and the
|
||||||
|
container starts persistent; the next start tries again. hive-priv checks a file
|
||||||
|
that fails to clone against the 2 GiB reserve again before it copies the data. A
|
||||||
|
salvage that fails partway removes the directory it created, and leaves the root
|
||||||
|
and any earlier salvage as they were. After a salvage, hive-priv logs `salvage: <container> <size> reflinked=<n>/<m> files`.
|
||||||
|
|
||||||
|
**Concurrency.** hive-priv holds an `flock` on
|
||||||
|
`/run/hive-priv/salvage-<container>.lock` (the unit's `RuntimeDirectory`, which
|
||||||
|
no container can write) for the whole `WriteNspawnFlags`, so a second request for
|
||||||
|
the same container waits and then finds the finished template. Across
|
||||||
|
containers, one salvage copies at a time, so the free space each one checks
|
||||||
|
excludes any copy still in flight. The request runs on the blocking
|
||||||
|
thread pool, so a long copy holds no async worker. hive-c0re's per-agent lease
|
||||||
|
already serialises the requests it sends; the lock covers hive-priv's own
|
||||||
|
guarantee.
|
||||||
|
|
||||||
The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`)
|
The journal still ships. The agent's collector (`nix/agent-modules/otel.nix`)
|
||||||
reads the guest's own `/var/log/journal` and pushes it over OTLP, so no
|
reads the guest's own `/var/log/journal` and pushes it over OTLP, so no
|
||||||
journal link to the host takes part; nspawn skips `--link-journal=try-guest`
|
journal link to the host takes part; nspawn skips `--link-journal=try-guest`
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ workspace = true
|
||||||
anyhow.workspace = true
|
anyhow.workspace = true
|
||||||
hive-priv-sock.workspace = true
|
hive-priv-sock.workspace = true
|
||||||
libc.workspace = true
|
libc.workspace = true
|
||||||
|
rustix.workspace = true
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,32 @@ pub(crate) fn handle_write_nspawn_flags(
|
||||||
Ok((String::new(), String::new()))
|
Ok((String::new(), String::new()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// [`handle_write_nspawn_flags`] on the blocking pool. The root salvage inside
|
||||||
|
/// it copies files for as long as the root is big, and would hold a runtime
|
||||||
|
/// worker for that time.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// As [`handle_write_nspawn_flags`], or when the task fails to join.
|
||||||
|
pub(crate) async fn handle_write_nspawn_flags_blocking(
|
||||||
|
container: String,
|
||||||
|
binds: Vec<BindMount>,
|
||||||
|
isolation: NetworkIsolation,
|
||||||
|
load_credentials: Vec<CredentialMount>,
|
||||||
|
hive_facts: HiveFacts,
|
||||||
|
) -> Result<(String, String)> {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
handle_write_nspawn_flags(
|
||||||
|
&container,
|
||||||
|
&binds,
|
||||||
|
&isolation,
|
||||||
|
&load_credentials,
|
||||||
|
&hive_facts,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.context("write nspawn flags task")?
|
||||||
|
}
|
||||||
|
|
||||||
/// Validate a system-level container name (`h-` prefix for all agents,
|
/// Validate a system-level container name (`h-` prefix for all agents,
|
||||||
/// the manager included) and return the logical agent name after it.
|
/// the manager included) and return the logical agent name after it.
|
||||||
/// Anything else is refused, the swarm's forge and matrix containers
|
/// Anything else is refused, the swarm's forge and matrix containers
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,9 @@
|
||||||
//! dispatch table. Each op family's handlers and validation live in their
|
//! dispatch table. Each op family's handlers and validation live in their
|
||||||
//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
|
//! own module: [`container`] (lifecycle), [`container_config`] (nspawn flags,
|
||||||
//! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
|
//! systemd drop-ins), [`rootfs`] (the salvage of a container's root before it
|
||||||
//! turns ephemeral), [`prestart`] (the check of a container's root that runs
|
//! turns ephemeral, with [`salvage_io`] for its space, reflink and lock calls),
|
||||||
//! before each start, as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
|
//! [`prestart`] (the check of a container's root that runs before each start,
|
||||||
|
//! as a sub-command), [`storage`] (btrfs subvolumes, quotas, snapshots, disk
|
||||||
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
|
//! measurement), [`files`] (pause marker, socket dir, legacy tmpfiles),
|
||||||
//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
|
//! [`infra`] (gateway nginx, infra containers) and [`closure`] (downloads from
|
||||||
//! the swarm cache). Helpers more than one family uses are in [`shared`].
|
//! the swarm cache). Helpers more than one family uses are in [`shared`].
|
||||||
|
|
@ -33,6 +34,7 @@ mod files;
|
||||||
mod infra;
|
mod infra;
|
||||||
mod prestart;
|
mod prestart;
|
||||||
mod rootfs;
|
mod rootfs;
|
||||||
|
mod salvage_io;
|
||||||
mod shared;
|
mod shared;
|
||||||
mod storage;
|
mod storage;
|
||||||
|
|
||||||
|
|
@ -383,18 +385,21 @@ async fn exec(
|
||||||
PrivRequest::ListContainers => container::container_run(&["list"]).await,
|
PrivRequest::ListContainers => container::container_run(&["list"]).await,
|
||||||
|
|
||||||
PrivRequest::WriteNspawnFlags {
|
PrivRequest::WriteNspawnFlags {
|
||||||
ref container,
|
|
||||||
ref binds,
|
|
||||||
ref isolation,
|
|
||||||
ref load_credentials,
|
|
||||||
ref hive_facts,
|
|
||||||
} => container_config::handle_write_nspawn_flags(
|
|
||||||
container,
|
container,
|
||||||
binds,
|
binds,
|
||||||
isolation,
|
isolation,
|
||||||
load_credentials,
|
load_credentials,
|
||||||
hive_facts,
|
hive_facts,
|
||||||
),
|
} => {
|
||||||
|
container_config::handle_write_nspawn_flags_blocking(
|
||||||
|
container,
|
||||||
|
binds,
|
||||||
|
isolation,
|
||||||
|
load_credentials,
|
||||||
|
hive_facts,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),
|
PrivRequest::EnsureAgentSocketDir { ref name } => files::ensure_agent_socket_dir(name),
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
84
hive-priv/src/salvage_io.rs
Normal file
84
hive-priv/src/salvage_io.rs
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
//! What the root salvage ([`crate::rootfs`]) asks of the host: free space,
|
||||||
|
//! reflinks, and the locks that keep two salvages apart.
|
||||||
|
//!
|
||||||
|
//! The space and reflink calls sit behind [`Backend`] so a test can set the
|
||||||
|
//! free space and make a clone fail.
|
||||||
|
|
||||||
|
use std::fs::{File, OpenOptions};
|
||||||
|
use std::io;
|
||||||
|
use std::os::unix::fs::OpenOptionsExt as _;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use anyhow::{Context as _, Result};
|
||||||
|
|
||||||
|
/// Where the per-container lock files live. A root-owned runtime directory
|
||||||
|
/// (`RuntimeDirectory=` of `hive-priv.service`), so no container can write it.
|
||||||
|
pub(crate) const LOCK_DIR: &str = "/run/hive-priv";
|
||||||
|
|
||||||
|
/// Held by the one salvage that is copying data at any moment.
|
||||||
|
pub(crate) static ONE_AT_A_TIME: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
|
/// Size and free space of a filesystem, in bytes. `free` is what an
|
||||||
|
/// unprivileged writer could still use.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub(crate) struct FsSpace {
|
||||||
|
pub(crate) free: u64,
|
||||||
|
pub(crate) total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The host calls the salvage makes.
|
||||||
|
pub(crate) trait Backend {
|
||||||
|
/// Space on the filesystem holding `path`.
|
||||||
|
fn space(&self, path: &Path) -> io::Result<FsSpace>;
|
||||||
|
|
||||||
|
/// Make `dst`, an empty file, share `src`'s data blocks. An error leaves
|
||||||
|
/// `dst` as it was: the filesystem cannot clone, or `src` and `dst` are on
|
||||||
|
/// different ones.
|
||||||
|
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The real filesystem: `statvfs(2)` and `ioctl(FICLONE)`.
|
||||||
|
pub(crate) struct Os;
|
||||||
|
|
||||||
|
impl Backend for Os {
|
||||||
|
fn space(&self, path: &Path) -> io::Result<FsSpace> {
|
||||||
|
let st = rustix::fs::statvfs(path)?;
|
||||||
|
Ok(FsSpace {
|
||||||
|
free: st.f_bavail.saturating_mul(st.f_frsize),
|
||||||
|
total: st.f_blocks.saturating_mul(st.f_frsize),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clone_file(&self, src: &File, dst: &File) -> io::Result<()> {
|
||||||
|
Ok(rustix::fs::ioctl_ficlone(dst, src)?)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An exclusive `flock(2)` on a container's lock file, released on drop.
|
||||||
|
pub(crate) struct ContainerLock {
|
||||||
|
_file: File,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ContainerLock {
|
||||||
|
/// Wait for the lock of `name` in `dir`, creating the file if needed.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error when the directory or file cannot be created or the
|
||||||
|
/// lock call fails.
|
||||||
|
pub(crate) fn acquire(dir: &Path, name: &str) -> Result<Self> {
|
||||||
|
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
|
||||||
|
let path = dir.join(format!("salvage-{name}.lock"));
|
||||||
|
let file = OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.truncate(false)
|
||||||
|
.mode(0o600)
|
||||||
|
.custom_flags(libc::O_NOFOLLOW)
|
||||||
|
.open(&path)
|
||||||
|
.with_context(|| format!("open {}", path.display()))?;
|
||||||
|
file.lock()
|
||||||
|
.with_context(|| format!("lock {}", path.display()))?;
|
||||||
|
Ok(Self { _file: file })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -146,6 +146,10 @@ in
|
||||||
];
|
];
|
||||||
# Writable HOME for nix's caches (see environment.HOME above).
|
# Writable HOME for nix's caches (see environment.HOME above).
|
||||||
StateDirectory = "hive-priv";
|
StateDirectory = "hive-priv";
|
||||||
|
# /run/hive-priv: the per-container salvage lock files. Root-owned and
|
||||||
|
# outside every path a container can write.
|
||||||
|
RuntimeDirectory = "hive-priv";
|
||||||
|
RuntimeDirectoryMode = "0700";
|
||||||
# With ProtectSystem=strict the root filesystem is read-only inside
|
# With ProtectSystem=strict the root filesystem is read-only inside
|
||||||
# hive-priv. When `nixos-container create/update` invokes nix, nix
|
# hive-priv. When `nixos-container create/update` invokes nix, nix
|
||||||
# creates a temporary result symlink in its working directory. Without
|
# creates a temporary result symlink in its working directory. Without
|
||||||
|
|
|
||||||
|
|
@ -78,6 +78,9 @@ pkgs.testers.runNixOSTest {
|
||||||
pkgs.hello
|
pkgs.hello
|
||||||
pkgs.tree
|
pkgs.tree
|
||||||
];
|
];
|
||||||
|
# The salvage skips when a copy would leave under 15% of the disk free, and
|
||||||
|
# this disk cannot clone files; the default 1 GiB leaves too little room.
|
||||||
|
virtualisation.diskSize = 4096;
|
||||||
# The agent bridge hive-priv names in the container's conf.
|
# The agent bridge hive-priv names in the container's conf.
|
||||||
networking.bridges.${bridge}.interfaces = [ ];
|
networking.bridges.${bridge}.interfaces = [ ];
|
||||||
networking.interfaces.${bridge}.ipv4.addresses = [
|
networking.interfaces.${bridge}.ipv4.addresses = [
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue