vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

a failed container start shows the unit's own journal lines in the op output

Browse source
This commit is contained in:
atlas 2026-11-01 16:33:26 +01:00 • committed by mara
commit 9a773349a8
7 changed files with 203 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/trust-boundary/security.md
View file

@ -256,6 +256,7 @@ known operations; there is no arbitrary command pass-through:
| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `StartContainer` / `StopContainer` | `nixos-container start/stop <name>` |
| `KillContainer` | `machinectl kill <machine> --signal=SIGKILL` (`nixos-container` has no kill verb) |
| `StartFailureLog` | `journalctl _SYSTEMD_UNIT=container@h-<name>.service --since @<t> -n 30 -o cat --no-pager`, fixed argv, 5 s timeout, output capped at 8 KiB; hive-priv clamps `<t>` to the last hour. hive-c0re appends the lines to the failed start's build log, which only the dashboard serves. |
| `CreateContainer` / `UpdateContainer` | `nixos-container create <name> --system-path <toplevel>`; for an update, `nix-env -p <profile> --set <toplevel>` and a reload of the unit. `<toplevel>` must be the store path the last successful `RealiseClosure` returned for that agent (in memory, per agent), so hive-c0re can't name a path never realised for the agent; hive-priv builds nothing. The binding doesn't mean hive-c0re can't choose the closure: it can still ask hive-priv to download any swarm-signed path for any agent and then apply it. The swarm key and the cache bound that to a cross-assignment risk (one agent gets another's closure), not an arbitrary-path one |
| `RealiseClosure` | `nix-store --realise <store_path> --max-jobs 0` against the swarm cache, with the host's configured cache address, and the public key and netrc from root-owned `/run/hyperhive-priv/attic/`, as per-call settings that replace the host's substituters and trusted keys and require a signature. The path must be `/nix/store/<32 chars>-<name>` and the agent name valid. It downloads and never builds |
| `DestroyContainer` | `nixos-container destroy <name>` |

15
hive-c0re/src/lifecycle/mod.rs
View file

@ -799,16 +799,23 @@ fn make_log_callback(
/// Put a failed non-streaming op's error into its build-log row, which has no
/// other output: `start` runs through `call`, not the streaming protocol. The
/// error carries `nixos-container`'s stderr.
/// error carries `nixos-container`'s stderr; `journal` is the container
/// unit's own log from hive-priv. An `Err` there (an older hive-priv does not
/// know the op) leaves the stderr alone.
fn record_failure(
logs: Option<&crate::build_logs::BuildLogs>,
log_id: Option<i64>,
err: &anyhow::Error,
journal: Result<String>,
) {
if let (Some(h), Some(id)) = (logs, log_id) {
for line in format!("{err:#}").lines() {
h.append_stderr(id, line);
}
match journal {
Ok(text) => text.lines().for_each(|l| h.append_stderr(id, l)),
Err(e) => tracing::debug!(error = ?e, "start journal unavailable"),
}
}
}
@ -879,9 +886,13 @@ async fn priv_run_inner(
None => Err(missing_toplevel(kind, name)),
},
"start" => {
let attempt = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
let started = crate::priv_client::start_container(name).await;
if let Err(e) = &started {
record_failure(logs.as_deref(), log_id, e);
let journal = crate::priv_client::start_failure_log(name, attempt).await;
record_failure(logs.as_deref(), log_id, e, journal);
}
started
}

30
hive-c0re/src/lifecycle/tests.rs
View file

@ -529,9 +529,33 @@ fn a_failed_start_puts_its_error_in_the_op_output() {
.start("updo", "start", "nixos-container start h-updo", None)
.expect("start");
let err = anyhow::anyhow!("nixos-container start h-updo failed (exit status: 1): Job failed");
record_failure(Some(&logs), Some(id), &err);
let journal = Ok("Creating mount point failed: Read-only file system".to_owned());
record_failure(Some(&logs), Some(id), &err, journal);
let full = logs.get_full(id).expect("get").expect("row");
assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
record_failure(Some(&logs), None, &err);
record_failure(None, Some(id), &err);
assert!(
full.stderr.contains("Read-only file system"),
"{}",
full.stderr
);
record_failure(Some(&logs), None, &err, Ok(String::new()));
record_failure(None, Some(id), &err, Ok(String::new()));
}
/// An older hive-priv rejects the op; the start's own error is still recorded.
#[test]
fn an_unknown_journal_op_keeps_the_start_error() {
let dir = tempfile::tempdir().expect("tempdir");
let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
let id = logs
.start("updo", "start", "nixos-container start h-updo", None)
.expect("start");
let err = anyhow::anyhow!("Job failed");
let unknown = Err(anyhow::anyhow!(
"parse request: unknown variant `start_failure_log`"
));
record_failure(Some(&logs), Some(id), &err, unknown);
let full = logs.get_full(id).expect("get").expect("row");
assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
assert!(!full.stderr.contains("unknown variant"), "{}", full.stderr);
}

13
hive-c0re/src/priv_client.rs
View file

@ -216,6 +216,19 @@ pub async fn start_container(name: &str) -> Result<()> {
.await?)
}
/// The container unit's journal lines since `since_unix`. An older hive-priv
/// answers an error (it does not know the op).
pub async fn start_failure_log(name: &str, since_unix: u64) -> Result<String> {
let (stdout, _) = check(
call(&PrivRequest::StartFailureLog {
name: name.to_owned(),
since_unix,
})
.await?,
)?;
Ok(stdout)
}
pub async fn stop_container(name: &str) -> Result<()> {
ok(call(&PrivRequest::StopContainer {
name: name.to_owned(),

5
hive-priv-sock/src/lib.rs
View file

@ -255,6 +255,11 @@ pub enum PrivRequest {
/// `nixos-container start <name>`
StartContainer { name: String },
/// The last journal lines of `name`'s own `container@h-<name>.service`
/// unit since `since_unix` (seconds), returned in `stdout`. hive-priv
/// builds the unit name and clamps `since_unix` to the last hour.
StartFailureLog { name: String, since_unix: u64 },
/// `nixos-container stop <name>`
StopContainer { name: String },

139
hive-priv/src/container.rs
View file

@ -49,6 +49,80 @@ pub(crate) async fn start_container(name: &str) -> Result<(String, String)> {
container_run(&["start", &machine]).await
}
/// Most bytes of journal text `StartFailureLog` returns.
const START_LOG_CAP: usize = 8 * 1024;
/// Longest `journalctl` run.
const START_LOG_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
/// How far back `StartFailureLog` may look.
const START_LOG_MAX_LOOKBACK_SECS: u64 = 3600;
/// The fixed `journalctl` argv for `machine`'s start log. The match is
/// `_SYSTEMD_UNIT`, so only lines written by the container unit's own
/// processes come back, not systemd's job messages about it.
fn start_log_argv(machine: &str, since: u64) -> Vec<String> {
[
&format!("_SYSTEMD_UNIT=container@{machine}.service"),
"--since",
&format!("@{since}"),
"-n",
"30",
"-o",
"cat",
"--no-pager",
]
.map(str::to_owned)
.to_vec()
}
/// The last `cap` bytes of `text`, cut at a char boundary.
fn tail_bytes(text: &str, cap: usize) -> &str {
let mut start = text.len().saturating_sub(cap);
while !text.is_char_boundary(start) {
start += 1;
}
&text[start..]
}
/// `StartFailureLog`: the container unit's recent journal lines, capped.
/// `run` executes `journalctl` with the argv it is given.
async fn start_failure_log_with<F, Fut>(
name: &str,
since_unix: u64,
now: u64,
run: F,
) -> Result<(String, String)>
where
F: FnOnce(Vec<String>) -> Fut,
Fut: std::future::Future<Output = Result<String>>,
{
validate_agent_name(name)?;
let since = since_unix.clamp(now.saturating_sub(START_LOG_MAX_LOOKBACK_SECS), now);
let out = run(start_log_argv(&container_system_name(name), since)).await?;
Ok((tail_bytes(&out, START_LOG_CAP).to_owned(), String::new()))
}
/// `StartFailureLog` against the real `journalctl`.
pub(crate) async fn start_failure_log(name: &str, since_unix: u64) -> Result<(String, String)> {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
start_failure_log_with(name, since_unix, now, |argv| async move {
let child = Command::new("journalctl")
.args(&argv)
.kill_on_drop(true)
.output();
let out = tokio::time::timeout(START_LOG_TIMEOUT, child)
.await
.context("journalctl timed out")?
.context("invoke journalctl")?;
if !out.status.success() {
bail!("journalctl failed ({})", out.status);
}
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
})
.await
}
/// `StopContainer`.
pub(crate) async fn stop_container(name: &str) -> Result<(String, String)> {
validate_agent_name(name)?;
@ -635,3 +709,68 @@ mod tests {
}
}
}
#[cfg(test)]
mod start_log_tests {
use super::*;
const NOW: u64 = 1_000_000;
async fn argv_for(name: &str, since: u64) -> Vec<String> {
let seen = std::sync::Mutex::new(Vec::new());
start_failure_log_with(name, since, NOW, |argv| {
*seen.lock().unwrap() = argv;
async { Ok(String::new()) }
})
.await
.unwrap();
seen.into_inner().unwrap()
}
#[tokio::test]
async fn odd_names_are_refused_before_journalctl_runs() {
for bad in ["", "../x", "a b", "A", "x.service", "x\n--vacuum-size=1"] {
let r =
start_failure_log_with(bad, NOW, NOW, |_| async { panic!("runner must not run") })
.await;
assert!(r.is_err(), "{bad:?} accepted");
}
}
#[tokio::test]
async fn the_argv_is_fixed_and_the_unit_is_built_here() {
assert_eq!(
argv_for("bob", NOW - 5).await,
[
"_SYSTEMD_UNIT=container@h-bob.service",
"--since",
"@999995",
"-n",
"30",
"-o",
"cat",
"--no-pager"
]
);
}
#[tokio::test]
async fn since_is_clamped_to_the_last_hour_and_not_the_future() {
assert_eq!(argv_for("bob", 0).await[2], format!("@{}", NOW - 3600));
assert_eq!(argv_for("bob", NOW + 99).await[2], format!("@{NOW}"));
}
#[tokio::test]
async fn the_output_is_capped_keeping_the_newest_bytes() {
let big = format!("{}end\n", "é".repeat(START_LOG_CAP));
let (out, _) = start_failure_log_with("bob", NOW, NOW, |_| async { Ok(big) })
.await
.unwrap();
assert!(out.len() <= START_LOG_CAP, "{}", out.len());
assert!(out.ends_with("end\n"));
let (small, _) = start_failure_log_with("bob", NOW, NOW, |_| async { Ok("a\n".into()) })
.await
.unwrap();
assert_eq!(small, "a\n");
}
}

5
hive-priv/src/main.rs
View file

@ -360,6 +360,11 @@ async fn exec(
match req {
PrivRequest::StartContainer { ref name } => container::start_container(name).await,
PrivRequest::StartFailureLog {
ref name,
since_unix,
} => container::start_failure_log(name, since_unix).await,
PrivRequest::StopContainer { ref name } => container::stop_container(name).await,
PrivRequest::KillContainer { ref name } => container::kill_container(name).await,