remove what no longer has a reader: build-log store, dashboard sent/delivered events, graph resource dep
This commit is contained in:
parent
32f5b74ba4
commit
49266e4410
2 changed files with 204 additions and 1054 deletions
|
|
@ -222,7 +222,7 @@ pub async fn provision_container(name: &str, paths: &AgentPaths) -> Result<()> {
|
|||
/// Assumes `provision_container` already registered the agent.
|
||||
/// `toplevel` is the store path of the system closure to create it from.
|
||||
pub async fn create_only(name: &str, toplevel: &str) -> Result<()> {
|
||||
priv_run_inner("create", name, Some(toplevel), None).await
|
||||
priv_run_inner("create", name, Some(toplevel)).await
|
||||
}
|
||||
|
||||
/// Rebuild-path preamble, run by the job queue's `Prebuild` node: fail fast on
|
||||
|
|
@ -252,10 +252,9 @@ pub async fn swap_update(
|
|||
hive: &HiveEnv,
|
||||
paths: &AgentPaths,
|
||||
toplevel: &str,
|
||||
node_id: Option<u64>,
|
||||
) -> Result<()> {
|
||||
write_dropins(name, hive, paths).await?;
|
||||
priv_run_inner("update", name, Some(toplevel), node_id).await
|
||||
priv_run_inner("update", name, Some(toplevel)).await
|
||||
}
|
||||
|
||||
/// Build the `AgentSpec` list for [`crate::agent_set::refresh`] from
|
||||
|
|
@ -728,22 +727,19 @@ fn is_actionable_nix_line(line: &str) -> bool {
|
|||
|
||||
/// What every log line of one container operation carries, so the swarm log
|
||||
/// store can group a build's lines and the builds board can list the builds:
|
||||
/// `build_id` (the build-log row's id, empty when the store is not installed),
|
||||
/// `agent` and `kind`. A `start` line opens the build and a `finish` line
|
||||
/// closes it with its `status` and `runtime_secs`; the output lines between
|
||||
/// them have no `event`.
|
||||
#[derive(Clone)]
|
||||
struct BuildTag {
|
||||
build_id: String,
|
||||
agent: String,
|
||||
kind: String,
|
||||
cmdline: String,
|
||||
}
|
||||
|
||||
impl BuildTag {
|
||||
fn new(log_id: Option<i64>, agent: &str, kind: &str, cmdline: &str) -> Self {
|
||||
fn new(agent: &str, kind: &str, cmdline: &str) -> Self {
|
||||
Self {
|
||||
build_id: log_id.map(|id| id.to_string()).unwrap_or_default(),
|
||||
agent: agent.to_owned(),
|
||||
kind: kind.to_owned(),
|
||||
cmdline: cmdline.to_owned(),
|
||||
|
|
@ -754,7 +750,6 @@ impl BuildTag {
|
|||
tracing::info!(
|
||||
target: "nixos-container",
|
||||
event = "start",
|
||||
build_id = %self.build_id,
|
||||
agent = %self.agent,
|
||||
kind = %self.kind,
|
||||
cmdline = %self.cmdline,
|
||||
|
|
@ -771,7 +766,6 @@ impl BuildTag {
|
|||
if actionable {
|
||||
tracing::warn!(
|
||||
target: "nixos-container",
|
||||
build_id = %self.build_id,
|
||||
agent = %self.agent,
|
||||
kind = %self.kind,
|
||||
cmdline = %self.cmdline,
|
||||
|
|
@ -780,7 +774,6 @@ impl BuildTag {
|
|||
} else {
|
||||
tracing::info!(
|
||||
target: "nixos-container",
|
||||
build_id = %self.build_id,
|
||||
agent = %self.agent,
|
||||
kind = %self.kind,
|
||||
cmdline = %self.cmdline,
|
||||
|
|
@ -795,7 +788,6 @@ impl BuildTag {
|
|||
tracing::info!(
|
||||
target: "nixos-container",
|
||||
event = "finish",
|
||||
build_id = %self.build_id,
|
||||
agent = %self.agent,
|
||||
kind = %self.kind,
|
||||
status,
|
||||
|
|
@ -808,7 +800,6 @@ impl BuildTag {
|
|||
tracing::warn!(
|
||||
target: "nixos-container",
|
||||
event = "finish",
|
||||
build_id = %self.build_id,
|
||||
agent = %self.agent,
|
||||
kind = %self.kind,
|
||||
status,
|
||||
|
|
@ -822,46 +813,22 @@ impl BuildTag {
|
|||
}
|
||||
|
||||
/// Build the per-line callback for `create_container_streaming` /
|
||||
/// `update_container_streaming`. Both ops share identical dispatch logic
|
||||
/// (log the line through its `BuildTag`, then `append_stdout` or
|
||||
/// `append_stderr`); this helper avoids duplicating that match body across the
|
||||
/// two call sites.
|
||||
fn make_log_callback(
|
||||
logs: Option<std::sync::Arc<crate::build_logs::BuildLogs>>,
|
||||
log_id: Option<i64>,
|
||||
tag: BuildTag,
|
||||
) -> impl FnMut(hive_priv_sock::PrivStream, &str) {
|
||||
use hive_priv_sock::PrivStream;
|
||||
move |stream, line| {
|
||||
tag.line(stream, line);
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
match stream {
|
||||
PrivStream::Stdout => h.append_stdout(id, line),
|
||||
PrivStream::Stderr => h.append_stderr(id, line),
|
||||
}
|
||||
}
|
||||
}
|
||||
/// `update_container_streaming`: log each line through its `BuildTag`.
|
||||
fn make_log_callback(tag: BuildTag) -> impl FnMut(hive_priv_sock::PrivStream, &str) {
|
||||
move |stream, line| tag.line(stream, line)
|
||||
}
|
||||
|
||||
/// Put a failed non-streaming op's error into its build-log row, which has no
|
||||
/// other output: `start` runs through `call`, not the streaming protocol. The
|
||||
/// error carries `nixos-container`'s stderr; `journal` is the container
|
||||
/// unit's own log from hive-priv. An `Err` there (an older hive-priv does not
|
||||
/// know the op) leaves the stderr alone.
|
||||
fn record_failure(
|
||||
logs: Option<&crate::build_logs::BuildLogs>,
|
||||
log_id: Option<i64>,
|
||||
err: &anyhow::Error,
|
||||
journal: Result<String>,
|
||||
) {
|
||||
if let (Some(h), Some(id)) = (logs, log_id) {
|
||||
for line in format!("{err:#}").lines() {
|
||||
h.append_stderr(id, line);
|
||||
}
|
||||
match journal {
|
||||
Ok(text) => text.lines().for_each(|l| h.append_stderr(id, l)),
|
||||
Err(e) => tracing::debug!(error = ?e, "start journal unavailable"),
|
||||
}
|
||||
/// Log a failed `start`'s error and the container unit's own journal (from
|
||||
/// hive-priv) line by line. An `Err` journal (an older hive-priv does not know
|
||||
/// the op) leaves just the error.
|
||||
fn record_failure(tag: &BuildTag, err: &anyhow::Error, journal: Result<String>) {
|
||||
use hive_priv_sock::PrivStream;
|
||||
for line in format!("{err:#}").lines() {
|
||||
tag.line(PrivStream::Stderr, line);
|
||||
}
|
||||
match journal {
|
||||
Ok(text) => text.lines().for_each(|l| tag.line(PrivStream::Stderr, l)),
|
||||
Err(e) => tracing::debug!(error = ?e, "start journal unavailable"),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -872,42 +839,21 @@ fn missing_toplevel(kind: &str, name: &str) -> anyhow::Error {
|
|||
anyhow::anyhow!("{name}: {kind} needs a closure to apply")
|
||||
}
|
||||
|
||||
/// Execute a container operation via hive-priv and integrate with
|
||||
/// `build_logs.sqlite`. hive-priv runs as root and forwards output lines
|
||||
/// to hive-c0re in real time via the streaming priv protocol. Each line
|
||||
/// is appended to the build-log row as it arrives, so the dashboard
|
||||
/// shows live progress during long `nixos-container create` / `update` runs.
|
||||
/// Execute a container operation via hive-priv. hive-priv runs as root and
|
||||
/// forwards output lines to hive-c0re in real time via the streaming priv
|
||||
/// protocol; each line is logged through the op's `BuildTag`.
|
||||
async fn priv_run(kind: &str, name: &str) -> Result<()> {
|
||||
priv_run_inner(kind, name, None, None).await
|
||||
priv_run_inner(kind, name, None).await
|
||||
}
|
||||
|
||||
/// Like `priv_run` but stamps `node_id` onto the build-log row it opens, so
|
||||
/// the log can be found from the queue node.
|
||||
///
|
||||
/// This used to be a `Fn(i64)` callback that handed the row id *back* to the
|
||||
/// queue, which then held it in a side map. The row carries the link itself
|
||||
/// now, so the id only ever travels one way.
|
||||
async fn priv_run_inner(
|
||||
kind: &str,
|
||||
name: &str,
|
||||
toplevel: Option<&str>,
|
||||
node_id: Option<u64>,
|
||||
) -> Result<()> {
|
||||
/// [`priv_run`] with the closure a `create` or `update` applies.
|
||||
async fn priv_run_inner(kind: &str, name: &str, toplevel: Option<&str>) -> Result<()> {
|
||||
let container = container_name(name);
|
||||
let cmdline = format!("nixos-container {kind} {container}");
|
||||
|
||||
let logs = crate::build_logs::global();
|
||||
let log_id = logs.as_ref().and_then(|h| {
|
||||
h.start(name, kind, &cmdline, node_id)
|
||||
.map_err(|e| {
|
||||
tracing::warn!(error = ?e, "build_logs: start failed (priv_run log dropped)");
|
||||
})
|
||||
.ok()
|
||||
});
|
||||
|
||||
// For long-running ops use the streaming protocol so build_logs
|
||||
// receives lines in real time rather than as a batch at completion.
|
||||
let tag = BuildTag::new(log_id, name, kind, &cmdline);
|
||||
// Long-running ops use the streaming protocol so lines are logged in real
|
||||
// time rather than as a batch at completion.
|
||||
let tag = BuildTag::new(name, kind, &cmdline);
|
||||
tag.started();
|
||||
let began = std::time::Instant::now();
|
||||
let result: Result<()> = match kind {
|
||||
|
|
@ -916,7 +862,7 @@ async fn priv_run_inner(
|
|||
crate::priv_client::create_container_streaming(
|
||||
name,
|
||||
toplevel,
|
||||
make_log_callback(logs.clone(), log_id, tag.clone()),
|
||||
make_log_callback(tag.clone()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -927,7 +873,7 @@ async fn priv_run_inner(
|
|||
crate::priv_client::update_container_streaming(
|
||||
name,
|
||||
toplevel,
|
||||
make_log_callback(logs.clone(), log_id, tag.clone()),
|
||||
make_log_callback(tag.clone()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -940,7 +886,7 @@ async fn priv_run_inner(
|
|||
let started = crate::priv_client::start_container(name).await;
|
||||
if let Err(e) = &started {
|
||||
let journal = crate::priv_client::start_failure_log(name, attempt).await;
|
||||
record_failure(logs.as_deref(), log_id, e, journal);
|
||||
record_failure(&tag, e, journal);
|
||||
}
|
||||
started
|
||||
}
|
||||
|
|
@ -952,22 +898,6 @@ async fn priv_run_inner(
|
|||
|
||||
let succeeded = result.is_ok();
|
||||
tag.finished(succeeded, began.elapsed());
|
||||
if let (Some(h), Some(id)) = (&logs, log_id) {
|
||||
h.finish(
|
||||
id,
|
||||
if succeeded {
|
||||
crate::build_logs::BuildStatus::Ok
|
||||
} else {
|
||||
crate::build_logs::BuildStatus::Fail
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) => match log_id {
|
||||
Some(id) => bail!("{e:#}; see build log #{id}"),
|
||||
None => bail!("{e:#}"),
|
||||
},
|
||||
}
|
||||
result.map_err(|e| anyhow::anyhow!("{e:#}"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -292,45 +292,57 @@ fn hive_facts_leave_unset_values_out() {
|
|||
assert_eq!(facts.name, None);
|
||||
}
|
||||
|
||||
/// A failed `start` leaves its error in the op's build-log row; without it the
|
||||
/// row is empty. A missing row id records nothing and does not panic.
|
||||
/// A failed `start` logs its error and the container unit's journal, line by
|
||||
/// line, under the op's tag.
|
||||
#[test]
|
||||
fn a_failed_start_puts_its_error_in_the_op_output() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
|
||||
let id = logs
|
||||
.start("updo", "start", "nixos-container start h-updo", None)
|
||||
.expect("start");
|
||||
fn a_failed_start_logs_its_error_and_the_journal() {
|
||||
let events = Events::default();
|
||||
let _guard = tracing::subscriber::set_default(events.clone());
|
||||
let tag = BuildTag::new("updo", "start", "nixos-container start h-updo");
|
||||
let err = anyhow::anyhow!("nixos-container start h-updo failed (exit status: 1): Job failed");
|
||||
let journal = Ok("Creating mount point failed: Read-only file system".to_owned());
|
||||
record_failure(Some(&logs), Some(id), &err, journal);
|
||||
let full = logs.get_full(id).expect("get").expect("row");
|
||||
assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
|
||||
record_failure(&tag, &err, journal);
|
||||
let logged = events.0.lock().unwrap();
|
||||
let messages: Vec<_> = logged
|
||||
.iter()
|
||||
.filter_map(|(_, f)| field(f, "message"))
|
||||
.collect();
|
||||
assert_eq!(messages.len(), 2, "{messages:?}");
|
||||
assert!(messages[0].contains("Job failed"), "{messages:?}");
|
||||
assert!(
|
||||
full.stderr.contains("Read-only file system"),
|
||||
"{}",
|
||||
full.stderr
|
||||
messages[1].contains("Read-only file system"),
|
||||
"{messages:?}"
|
||||
);
|
||||
record_failure(Some(&logs), None, &err, Ok(String::new()));
|
||||
record_failure(None, Some(id), &err, Ok(String::new()));
|
||||
for (_, fields) in logged.iter() {
|
||||
assert_eq!(field(fields, "agent"), Some("updo"));
|
||||
}
|
||||
}
|
||||
|
||||
/// An older hive-priv rejects the op; the start's own error is still recorded.
|
||||
/// An older hive-priv rejects the journal op; the start's own error is still
|
||||
/// logged and the rejection is not.
|
||||
#[test]
|
||||
fn an_unknown_journal_op_keeps_the_start_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
|
||||
let id = logs
|
||||
.start("updo", "start", "nixos-container start h-updo", None)
|
||||
.expect("start");
|
||||
let events = Events::default();
|
||||
let _guard = tracing::subscriber::set_default(events.clone());
|
||||
let tag = BuildTag::new("updo", "start", "nixos-container start h-updo");
|
||||
let err = anyhow::anyhow!("Job failed");
|
||||
let unknown = Err(anyhow::anyhow!(
|
||||
"parse request: unknown variant `start_failure_log`"
|
||||
));
|
||||
record_failure(Some(&logs), Some(id), &err, unknown);
|
||||
let full = logs.get_full(id).expect("get").expect("row");
|
||||
assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
|
||||
assert!(!full.stderr.contains("unknown variant"), "{}", full.stderr);
|
||||
record_failure(&tag, &err, unknown);
|
||||
let logged = events.0.lock().unwrap();
|
||||
let messages: Vec<_> = logged
|
||||
.iter()
|
||||
.filter_map(|(_, f)| field(f, "message"))
|
||||
.collect();
|
||||
assert!(
|
||||
messages.iter().any(|m| m.contains("Job failed")),
|
||||
"{messages:?}"
|
||||
);
|
||||
assert!(
|
||||
!messages.iter().any(|m| m.contains("unknown variant")),
|
||||
"{messages:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// One logged event: its level and each field as `name=value`.
|
||||
|
|
@ -372,16 +384,16 @@ fn field<'a>(fields: &'a [(String, String)], name: &str) -> Option<&'a str> {
|
|||
.map(|(_, v)| v.as_str())
|
||||
}
|
||||
|
||||
/// A build's log lines say which build they belong to, so the log store can
|
||||
/// group them: a `start` line, the output lines, then a `finish` line with the
|
||||
/// A build's log lines carry its agent and kind, so the log store can group
|
||||
/// them: a `start` line, the output lines, then a `finish` line with the
|
||||
/// status and runtime. A nix error is a warning, chatter stays info, and a
|
||||
/// failed finish is a warning.
|
||||
#[test]
|
||||
fn a_build_logs_its_start_its_lines_and_its_finish_under_one_build_id() {
|
||||
fn a_build_logs_its_start_its_lines_and_its_finish() {
|
||||
use hive_priv_sock::PrivStream;
|
||||
let events = Events::default();
|
||||
let _guard = tracing::subscriber::set_default(events.clone());
|
||||
let tag = BuildTag::new(Some(7), "alice", "update", "nixos-container update h-alice");
|
||||
let tag = BuildTag::new("alice", "update", "nixos-container update h-alice");
|
||||
tag.started();
|
||||
tag.line(PrivStream::Stdout, "copying 3 paths");
|
||||
tag.line(PrivStream::Stderr, "error: builder failed");
|
||||
|
|
@ -390,7 +402,6 @@ fn a_build_logs_its_start_its_lines_and_its_finish_under_one_build_id() {
|
|||
let logged = events.0.lock().unwrap();
|
||||
assert_eq!(logged.len(), 4);
|
||||
for (_, fields) in logged.iter() {
|
||||
assert_eq!(field(fields, "build_id"), Some("7"));
|
||||
assert_eq!(field(fields, "agent"), Some("alice"));
|
||||
assert_eq!(field(fields, "kind"), Some("update"));
|
||||
}
|
||||
|
|
@ -411,18 +422,6 @@ fn a_build_logs_its_start_its_lines_and_its_finish_under_one_build_id() {
|
|||
assert_eq!(field(&logged[3].1, "runtime_secs"), Some("12"));
|
||||
}
|
||||
|
||||
/// Without the build-log store there is no id; the lines still carry the
|
||||
/// agent and kind, with an empty `build_id`.
|
||||
#[test]
|
||||
fn a_build_without_a_log_row_has_an_empty_build_id() {
|
||||
let events = Events::default();
|
||||
let _guard = tracing::subscriber::set_default(events.clone());
|
||||
BuildTag::new(None, "alice", "create", "nixos-container create h-alice").started();
|
||||
let logged = events.0.lock().unwrap();
|
||||
assert_eq!(field(&logged[0].1, "build_id"), Some(""));
|
||||
assert_eq!(field(&logged[0].1, "agent"), Some("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stop_kind_is_leaked_once_per_agent() {
|
||||
let first = stop_kind("stopkind-a");
|
||||
|
|
|
|||
Loading…
Reference in a new issue