feat(#2470): capture config-approval eval/deploy nix runs in build logs
This commit is contained in:
parent
741852a8b4
commit
207b66e35d
1 changed files with 71 additions and 11 deletions
|
|
@ -238,7 +238,7 @@ pub async fn prepare_deploy(name: &str) -> Result<()> {
|
||||||
let _guard = META_LOCK.lock().await;
|
let _guard = META_LOCK.lock().await;
|
||||||
let dir = crate::paths::meta_root();
|
let dir = crate::paths::meta_root();
|
||||||
let input = format!("agent-{name}");
|
let input = format!("agent-{name}");
|
||||||
nix(&dir, &["flake", "update", &input]).await?;
|
nix_logged(&dir, &["flake", "update", &input], name, "prepare-deploy").await?;
|
||||||
// Stage the new lock — git+file://'s dirty-tree fetcher reads
|
// Stage the new lock — git+file://'s dirty-tree fetcher reads
|
||||||
// index entries, so the upcoming nixos-container update sees the
|
// index entries, so the upcoming nixos-container update sees the
|
||||||
// bumped rev without a commit yet.
|
// bumped rev without a commit yet.
|
||||||
|
|
@ -324,7 +324,7 @@ pub async fn verify_commit(name: &str, applied_dir: &Path, sha: &str) -> Result<
|
||||||
let input = format!("agent-{name}");
|
let input = format!("agent-{name}");
|
||||||
let over = agent_input_override(applied_dir, sha);
|
let over = agent_input_override(applied_dir, sha);
|
||||||
let attr = format!(".#nixosConfigurations.{name}.config.system.build.toplevel.drvPath");
|
let attr = format!(".#nixosConfigurations.{name}.config.system.build.toplevel.drvPath");
|
||||||
nix(
|
nix_logged(
|
||||||
&dir,
|
&dir,
|
||||||
&[
|
&[
|
||||||
"eval",
|
"eval",
|
||||||
|
|
@ -334,6 +334,8 @@ pub async fn verify_commit(name: &str, applied_dir: &Path, sha: &str) -> Result<
|
||||||
&over,
|
&over,
|
||||||
"--no-write-lock-file",
|
"--no-write-lock-file",
|
||||||
],
|
],
|
||||||
|
name,
|
||||||
|
"verify",
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
@ -1308,19 +1310,31 @@ async fn paths_dirty(dir: &Path, paths: &[&str]) -> Result<bool> {
|
||||||
Ok(!out.status.success())
|
Ok(!out.status.success())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
|
/// Full argv for a `nix` invocation, prefixed with the flakes-enabling
|
||||||
// `--extra-experimental-features` belt-and-suspenders for hosts
|
/// experimental-features flag. `--extra-experimental-features` is
|
||||||
// that haven't set this in nix.conf. The hyperhive module's
|
/// belt-and-suspenders for hosts that haven't set this in nix.conf: the
|
||||||
// deploy guide assumes flakes are already enabled, but the cost
|
/// hyperhive module's deploy guide assumes flakes are already enabled,
|
||||||
// of being defensive is one extra argv each call.
|
/// but the cost of being defensive is one extra argv each call.
|
||||||
|
fn nix_argv<'a>(args: &[&'a str]) -> Vec<&'a str> {
|
||||||
let mut all = vec!["--extra-experimental-features", "nix-command flakes"];
|
let mut all = vec!["--extra-experimental-features", "nix-command flakes"];
|
||||||
all.extend(args);
|
all.extend_from_slice(args);
|
||||||
let out = Command::new("nix")
|
all
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run `nix <args>` in `dir` (flakes enabled), capturing combined output.
|
||||||
|
/// Shared core of [`nix`] and [`nix_logged`].
|
||||||
|
async fn nix_output(dir: &Path, args: &[&str]) -> Result<std::process::Output> {
|
||||||
|
Command::new("nix")
|
||||||
.current_dir(dir)
|
.current_dir(dir)
|
||||||
.args(&all)
|
.args(nix_argv(args))
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))?;
|
.with_context(|| format!("nix {} in {}", args.join(" "), dir.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Turn a finished nix [`Output`](std::process::Output) into `Result<()>`,
|
||||||
|
/// bailing with the trimmed stderr tail on non-zero exit.
|
||||||
|
fn nix_check(args: &[&str], out: &std::process::Output) -> Result<()> {
|
||||||
if !out.status.success() {
|
if !out.status.success() {
|
||||||
bail!(
|
bail!(
|
||||||
"nix {} failed ({}): {}",
|
"nix {} failed ({}): {}",
|
||||||
|
|
@ -1332,6 +1346,52 @@ async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn nix(dir: &Path, args: &[&str]) -> Result<()> {
|
||||||
|
let out = nix_output(dir, args).await?;
|
||||||
|
nix_check(args, &out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like [`nix`] but records the invocation (cmdline + stdout + stderr +
|
||||||
|
/// terminal status) into a `build_logs.sqlite` row so a config-approval
|
||||||
|
/// eval/deploy step shows up on the dashboard. This is what makes a
|
||||||
|
/// failing eval-verify visible: it runs before any container build, so
|
||||||
|
/// without a row a rejected config approval leaves the operator with
|
||||||
|
/// zero build logs to look at. Best-effort logging: a missing global
|
||||||
|
/// handle or a failed `start()` just skips the row — the command still
|
||||||
|
/// runs and its exit status is still enforced.
|
||||||
|
async fn nix_logged(dir: &Path, args: &[&str], agent: &str, kind: &str) -> Result<()> {
|
||||||
|
let cmdline = format!("nix {}", nix_argv(args).join(" "));
|
||||||
|
let logs = crate::build_logs::global();
|
||||||
|
let log_id = logs.as_ref().and_then(|h| {
|
||||||
|
h.start(agent, kind, &cmdline)
|
||||||
|
.map_err(|e| {
|
||||||
|
tracing::warn!(error = ?e, %kind, "build_logs: start failed (meta log dropped)");
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
});
|
||||||
|
|
||||||
|
let out = nix_output(dir, args).await?;
|
||||||
|
|
||||||
|
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||||
|
for line in String::from_utf8_lossy(&out.stdout).lines() {
|
||||||
|
h.append_stdout(id, line);
|
||||||
|
}
|
||||||
|
for line in String::from_utf8_lossy(&out.stderr).lines() {
|
||||||
|
h.append_stderr(id, line);
|
||||||
|
}
|
||||||
|
h.finish(
|
||||||
|
id,
|
||||||
|
if out.status.success() {
|
||||||
|
crate::build_logs::BuildStatus::Ok
|
||||||
|
} else {
|
||||||
|
crate::build_logs::BuildStatus::Fail
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
nix_check(args, &out)
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue