deploy the closure the swarm declares, read no forge head, drop the update badge
This commit is contained in:
atlas
mara
parent
279428a66f
commit
47c2acb640
28 changed files with 647 additions and 2001 deletions
|
|
@ -33,15 +33,6 @@ pub fn read_at(path: &Path) -> std::io::Result<Stored> {
|
|||
super::read_map(path)
|
||||
}
|
||||
|
||||
/// `agent`'s archetype per the file at [`path`], `None` for a custom agent.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// As [`read_at`].
|
||||
pub fn of(agent: &str) -> std::io::Result<Option<ArchetypeRef>> {
|
||||
Ok(read_at(&path())?.remove(agent))
|
||||
}
|
||||
|
||||
/// Read `agent`'s key and store what it holds. No queue, or a bucket that
|
||||
/// does not answer, leaves the stored entry as it is.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
//! The closure the swarm built for an agent, as a deploy reads its record
|
||||
//! before downloading it.
|
||||
//! The closure the swarm declares for an agent: its record in `agent-closures`
|
||||
//! read before a deploy downloads it, and the marker of the store path the
|
||||
//! hive last applied, which a changed declaration is compared with.
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context as _, Result, anyhow};
|
||||
use swarm_wire::agent_closures::AgentClosure;
|
||||
|
||||
use super::probe;
|
||||
|
|
@ -17,3 +20,162 @@ pub async fn record(agent: &str) -> Result<Option<AgentClosure>> {
|
|||
.ok_or_else(|| anyhow!("this hive has no connection to the swarm queue"))?;
|
||||
probe::read_record(&client, agent).await
|
||||
}
|
||||
|
||||
/// Record that `agent` runs the closure at `store_path`.
|
||||
///
|
||||
/// # Errors
|
||||
/// The marker could not be written.
|
||||
pub fn mark_deployed(agent: &str, store_path: &str) -> Result<()> {
|
||||
mark_deployed_in(&crate::paths::applied_closure_marker(agent), store_path)
|
||||
}
|
||||
|
||||
/// The store path `agent` was last swapped to; `None` when the hive has no
|
||||
/// record of one: it never applied a closure, or last did before the marker
|
||||
/// existed.
|
||||
#[must_use]
|
||||
pub fn deployed(agent: &str) -> Option<String> {
|
||||
deployed_in(&crate::paths::applied_closure_marker(agent))
|
||||
}
|
||||
|
||||
/// Written whole to a temp file beside the marker and renamed over it, so a
|
||||
/// reader sees the old marker or the new one.
|
||||
fn mark_deployed_in(marker: &Path, store_path: &str) -> Result<()> {
|
||||
let tmp = marker.with_extension("tmp");
|
||||
if let Some(dir) = marker.parent() {
|
||||
std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
|
||||
}
|
||||
std::fs::write(&tmp, store_path).with_context(|| format!("write {}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, marker)
|
||||
.with_context(|| format!("rename {} to {}", tmp.display(), marker.display()))
|
||||
}
|
||||
|
||||
fn deployed_in(marker: &Path) -> Option<String> {
|
||||
let stored = std::fs::read_to_string(marker).ok()?;
|
||||
let stored = stored.trim();
|
||||
(!stored.is_empty()).then(|| stored.to_owned())
|
||||
}
|
||||
|
||||
/// Whether the swarm's declared closure is one the hive should now deploy: the
|
||||
/// builder pushed it and it differs from what `deployed` says runs.
|
||||
///
|
||||
/// An agent with no `deployed` marker is not pending, since the hive cannot
|
||||
/// tell what it runs: every agent starts that way when the hive first reads
|
||||
/// this marker, and deploying all of them at once is worse than waiting for the
|
||||
/// swarm's next deploy request, which writes the marker.
|
||||
#[must_use]
|
||||
pub fn pending(record: &AgentClosure, deployed: Option<&str>) -> bool {
|
||||
record.cached && deployed.is_some_and(|deployed| deployed != record.store_path)
|
||||
}
|
||||
|
||||
/// Whether `deployed` is the closure `record` declares, so deploying it again
|
||||
/// would change nothing.
|
||||
#[must_use]
|
||||
pub fn up_to_date(record: &AgentClosure, deployed: Option<&str>) -> bool {
|
||||
deployed == Some(record.store_path.as_str())
|
||||
}
|
||||
|
||||
/// The store path the swarm declares for `agent` when it is [`pending`]. A record
|
||||
/// that cannot be read, or none, is logged and counts as nothing pending: the
|
||||
/// hive then deploys nothing it was not told to.
|
||||
pub async fn pending_path(agent: &str) -> Option<String> {
|
||||
match record(agent).await {
|
||||
Ok(Some(record)) => {
|
||||
pending(&record, deployed(agent).as_deref()).then_some(record.store_path)
|
||||
}
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
tracing::warn!(%agent, error = %format!("{e:#}"), "cannot read the agent's declared closure");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the closure the swarm declares for `agent` is the one it runs; false
|
||||
/// when that cannot be told, so a deploy request goes ahead.
|
||||
pub async fn is_up_to_date(agent: &str) -> bool {
|
||||
match record(agent).await {
|
||||
Ok(Some(record)) => up_to_date(&record, deployed(agent).as_deref()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const DECLARED: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-alice";
|
||||
const RUNNING: &str = "/nix/store/zyxwvsrqpnmlkjihgfdcbadfhijklmn-nixos-system-alice";
|
||||
|
||||
fn declared(cached: bool) -> AgentClosure {
|
||||
AgentClosure {
|
||||
meta_rev: "a".repeat(40),
|
||||
store_path: DECLARED.to_owned(),
|
||||
system: "x86_64-linux".to_owned(),
|
||||
built_unix: 1,
|
||||
cached,
|
||||
private_users: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_declaration_other_than_the_running_closure_is_pending() {
|
||||
assert!(pending(&declared(true), Some(RUNNING)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_running_closure_is_not_pending() {
|
||||
assert!(!pending(&declared(true), Some(DECLARED)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_closure_the_builder_did_not_push_is_not_pending() {
|
||||
assert!(!pending(&declared(false), Some(RUNNING)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_declared_closure_is_up_to_date() {
|
||||
assert!(up_to_date(&declared(true), Some(DECLARED)));
|
||||
assert!(!up_to_date(&declared(true), Some(RUNNING)));
|
||||
assert!(!up_to_date(&declared(true), None), "unknown is deployed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_agent_with_no_marker_is_not_pending() {
|
||||
assert!(!pending(&declared(true), None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_marker_holds_the_store_path_and_is_replaced_whole() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let marker = dir.path().join("applied").join(".alice.closure");
|
||||
assert_eq!(deployed_in(&marker), None, "no marker yet");
|
||||
mark_deployed_in(&marker, RUNNING).unwrap();
|
||||
assert_eq!(deployed_in(&marker).as_deref(), Some(RUNNING));
|
||||
mark_deployed_in(&marker, DECLARED).unwrap();
|
||||
assert_eq!(deployed_in(&marker).as_deref(), Some(DECLARED));
|
||||
}
|
||||
|
||||
/// A hive that ran the previous marker format has a config commit in its
|
||||
/// old files and none at the new path: the agent is not pending, and the
|
||||
/// old files are not read or touched.
|
||||
#[test]
|
||||
fn markers_in_the_old_format_are_neither_read_nor_removed() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let old = dir.path().join(".alice.config-rev");
|
||||
std::fs::write(&old, "1111111111111111111111111111111111111111").unwrap();
|
||||
let marker = dir.path().join(".alice.closure");
|
||||
assert_eq!(deployed_in(&marker), None);
|
||||
assert!(!pending(&declared(true), deployed_in(&marker).as_deref()));
|
||||
assert!(old.exists());
|
||||
mark_deployed_in(&marker, DECLARED).unwrap();
|
||||
assert!(old.exists(), "a successful deploy leaves old markers alone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_marker_is_unknown() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let marker = dir.path().join(".alice.closure");
|
||||
std::fs::write(&marker, "\n").unwrap();
|
||||
assert_eq!(deployed_in(&marker), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@
|
|||
//! [`super::closure`].
|
||||
//!
|
||||
//! The verdict is the first of these that fails, in this order:
|
||||
//! no credentials (`no-credentials`), no record (`no-record`), a record built
|
||||
//! from another `internal/meta` commit than the head that pins the agent
|
||||
//! differently (`rev-mismatch`), a record
|
||||
//! no credentials (`no-credentials`), no record (`no-record`), a record
|
||||
//! the builder did not push (`not-cached`), the cache not holding the narinfo
|
||||
//! (`not-cached`), a request that failed or was refused (`fetch-failed`), a
|
||||
//! narinfo no signature of the key covers (`bad-signature`).
|
||||
|
|
@ -35,7 +33,6 @@ const FAILURES_BEFORE_WARNING: u32 = 3;
|
|||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Reason {
|
||||
NoRecord,
|
||||
RevMismatch,
|
||||
NotCached,
|
||||
NoCredentials,
|
||||
FetchFailed,
|
||||
|
|
@ -47,7 +44,6 @@ impl Reason {
|
|||
pub const fn label(self) -> &'static str {
|
||||
match self {
|
||||
Reason::NoRecord => "no-record",
|
||||
Reason::RevMismatch => "rev-mismatch",
|
||||
Reason::NotCached => "not-cached",
|
||||
Reason::NoCredentials => "no-credentials",
|
||||
Reason::FetchFailed => "fetch-failed",
|
||||
|
|
@ -84,26 +80,10 @@ impl Cache for Http {
|
|||
}
|
||||
}
|
||||
|
||||
/// `record` when it was built from the `internal/meta` commit `head`, or from
|
||||
/// one that pins the agent as `head` does (`same_pins`), and the builder pushed
|
||||
/// it to the cache.
|
||||
pub(crate) fn current_record<'a>(
|
||||
record: Option<&'a AgentClosure>,
|
||||
head: Option<&str>,
|
||||
same_pins: bool,
|
||||
) -> Result<&'a AgentClosure, Unpullable> {
|
||||
/// `record` when the builder pushed it to the cache.
|
||||
pub(crate) fn current_record(record: Option<&AgentClosure>) -> Result<&AgentClosure, Unpullable> {
|
||||
let record =
|
||||
record.ok_or_else(|| no(Reason::NoRecord, "no build is recorded for the agent"))?;
|
||||
if head != Some(record.meta_rev.as_str()) && !same_pins {
|
||||
return Err(no(
|
||||
Reason::RevMismatch,
|
||||
format!(
|
||||
"built from {}, internal/meta is at {}",
|
||||
record.meta_rev,
|
||||
head.unwrap_or("nothing")
|
||||
),
|
||||
));
|
||||
}
|
||||
if !record.cached {
|
||||
return Err(no(
|
||||
Reason::NotCached,
|
||||
|
|
@ -113,14 +93,10 @@ pub(crate) fn current_record<'a>(
|
|||
Ok(record)
|
||||
}
|
||||
|
||||
/// Whether the closure `record` names can be pulled with `facts`. `head` is the
|
||||
/// `internal/meta` commit a deploy would build from; `same_pins` is whether the
|
||||
/// record's commit pins the agent as `head` does.
|
||||
/// Whether the closure `record` names can be pulled with `facts`.
|
||||
pub(crate) async fn judge(
|
||||
facts: Option<&Facts>,
|
||||
record: Option<&AgentClosure>,
|
||||
head: Option<&str>,
|
||||
same_pins: bool,
|
||||
cache: &impl Cache,
|
||||
) -> Result<(), Unpullable> {
|
||||
let facts = facts.ok_or_else(|| {
|
||||
|
|
@ -129,7 +105,7 @@ pub(crate) async fn judge(
|
|||
"the swarm has given this hive no cache facts",
|
||||
)
|
||||
})?;
|
||||
let record = current_record(record, head, same_pins)?;
|
||||
let record = current_record(record)?;
|
||||
let text = match cache.narinfo(facts, &record.store_path).await {
|
||||
Ok(Some(text)) => text,
|
||||
Ok(None) => return Err(no(Reason::NotCached, "the cache does not hold the narinfo")),
|
||||
|
|
@ -182,7 +158,7 @@ pub async fn follow(client: async_nats::Client, mut shutdown: tokio::sync::watch
|
|||
/// One pass: publish the facts, then judge every agent.
|
||||
///
|
||||
/// # Errors
|
||||
/// A read that failed (the store, the queue, the forge), which leaves the last
|
||||
/// A read that failed (the store, the queue), which leaves the last
|
||||
/// verdicts as they are: the failure says nothing about any agent.
|
||||
async fn pass(client: &async_nats::Client) -> Result<()> {
|
||||
swarm_queue_client::ensure_connected(client)?;
|
||||
|
|
@ -190,10 +166,6 @@ async fn pass(client: &async_nats::Client) -> Result<()> {
|
|||
let agents = crate::lifecycle::agents_for_meta_listing()
|
||||
.await
|
||||
.context("enumerating this hive's agents")?;
|
||||
let head = match &facts {
|
||||
Some(_) => meta_head().await?,
|
||||
None => None,
|
||||
};
|
||||
let http = Http(narinfo::client()?);
|
||||
let mut now = Vec::with_capacity(agents.len());
|
||||
for spec in agents {
|
||||
|
|
@ -201,18 +173,7 @@ async fn pass(client: &async_nats::Client) -> Result<()> {
|
|||
Some(_) => read_record(client, &spec.name).await?,
|
||||
None => None,
|
||||
};
|
||||
let same_pins = match (&record, head.as_deref()) {
|
||||
(Some(r), Some(head)) => pins_unchanged(&spec.name, &r.meta_rev, head).await,
|
||||
_ => false,
|
||||
};
|
||||
let verdict = judge(
|
||||
facts.as_ref(),
|
||||
record.as_ref(),
|
||||
head.as_deref(),
|
||||
same_pins,
|
||||
&http,
|
||||
)
|
||||
.await;
|
||||
let verdict = judge(facts.as_ref(), record.as_ref(), &http).await;
|
||||
now.push((spec.name, verdict.as_ref().err().map(|u| u.reason)));
|
||||
report(&now[now.len() - 1], verdict.err().map(|u| u.detail));
|
||||
}
|
||||
|
|
@ -222,26 +183,6 @@ async fn pass(client: &async_nats::Client) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether `agent` has the same pins at `built` as at `head`. A commit that
|
||||
/// cannot be read counts as different, which leaves the verdict to the commit
|
||||
/// ids.
|
||||
async fn pins_unchanged(agent: &str, built: &str, head: &str) -> bool {
|
||||
if built == head {
|
||||
return true;
|
||||
}
|
||||
let read = async {
|
||||
let archetype = crate::archetypes::of(agent)?.is_some();
|
||||
let source = crate::forge::internal_meta_source()
|
||||
.await
|
||||
.context("no forge token")?;
|
||||
crate::meta_rev::pins_unchanged(&source, agent, archetype, built, head).await
|
||||
};
|
||||
read.await.unwrap_or_else(|e| {
|
||||
tracing::debug!(%agent, error = %format!("{e:#}"), "attic: cannot compare the agent's pins");
|
||||
false
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn read_record(
|
||||
client: &async_nats::Client,
|
||||
agent: &str,
|
||||
|
|
@ -256,19 +197,6 @@ pub(crate) async fn read_record(
|
|||
}
|
||||
}
|
||||
|
||||
/// The `internal/meta` head, or `None` when the swarm has not published it or
|
||||
/// this hive has no forge token.
|
||||
async fn meta_head() -> Result<Option<String>> {
|
||||
use crate::meta_rev::MetaSource as _;
|
||||
match crate::forge::internal_meta_source().await {
|
||||
Some(source) => source
|
||||
.meta_head()
|
||||
.await
|
||||
.context("reading internal/meta's head"),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The log line: at info when the verdict differs from the last pass, else
|
||||
/// debug.
|
||||
fn report((agent, verdict): &(String, Option<Reason>), detail: Option<String>) {
|
||||
|
|
@ -341,13 +269,9 @@ mod tests {
|
|||
async fn reason(
|
||||
facts: Option<&Facts>,
|
||||
record: Option<&AgentClosure>,
|
||||
head: Option<&str>,
|
||||
cache: Answers,
|
||||
) -> Option<Reason> {
|
||||
judge(facts, record, head, false, &cache)
|
||||
.await
|
||||
.err()
|
||||
.map(|u| u.reason)
|
||||
judge(facts, record, &cache).await.err().map(|u| u.reason)
|
||||
}
|
||||
|
||||
fn good() -> Answers {
|
||||
|
|
@ -355,73 +279,37 @@ mod tests {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_signed_narinfo_for_the_head_build_is_pullable() {
|
||||
assert_eq!(
|
||||
reason(Some(&facts()), Some(&record()), Some(REV), good()).await,
|
||||
None
|
||||
);
|
||||
async fn a_signed_narinfo_for_the_recorded_build_is_pullable() {
|
||||
assert_eq!(reason(Some(&facts()), Some(&record()), good()).await, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_facts_is_no_credentials_even_with_a_record() {
|
||||
let r = reason(None, Some(&record()), Some(REV), good()).await;
|
||||
let r = reason(None, Some(&record()), good()).await;
|
||||
assert_eq!(r, Some(Reason::NoCredentials));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_record_is_no_record() {
|
||||
let r = reason(Some(&facts()), None, Some(REV), good()).await;
|
||||
let r = reason(Some(&facts()), None, good()).await;
|
||||
assert_eq!(r, Some(Reason::NoRecord));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_record_from_another_commit_is_a_rev_mismatch() {
|
||||
let r = reason(Some(&facts()), Some(&record()), Some("other"), good()).await;
|
||||
assert_eq!(r, Some(Reason::RevMismatch));
|
||||
let r = reason(Some(&facts()), Some(&record()), None, good()).await;
|
||||
assert_eq!(r, Some(Reason::RevMismatch), "no head to compare with");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_record_from_another_commit_with_the_same_pins_is_pullable() {
|
||||
let (facts, record) = (facts(), record());
|
||||
let judged = judge(Some(&facts), Some(&record), Some("other"), true, &good()).await;
|
||||
assert!(judged.is_ok());
|
||||
let unpushed = AgentClosure {
|
||||
cached: false,
|
||||
..record
|
||||
};
|
||||
let judged = judge(Some(&facts), Some(&unpushed), Some("other"), true, &good()).await;
|
||||
assert_eq!(judged.err().map(|u| u.reason), Some(Reason::NotCached));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unpushed_record_or_a_missing_narinfo_is_not_cached() {
|
||||
let unpushed = AgentClosure {
|
||||
cached: false,
|
||||
..record()
|
||||
};
|
||||
let r = reason(Some(&facts()), Some(&unpushed), Some(REV), good()).await;
|
||||
let r = reason(Some(&facts()), Some(&unpushed), good()).await;
|
||||
assert_eq!(r, Some(Reason::NotCached));
|
||||
let r = reason(
|
||||
Some(&facts()),
|
||||
Some(&record()),
|
||||
Some(REV),
|
||||
Answers(Ok(None)),
|
||||
)
|
||||
.await;
|
||||
let r = reason(Some(&facts()), Some(&record()), Answers(Ok(None))).await;
|
||||
assert_eq!(r, Some(Reason::NotCached));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_request_is_fetch_failed() {
|
||||
let r = reason(
|
||||
Some(&facts()),
|
||||
Some(&record()),
|
||||
Some(REV),
|
||||
Answers(Err("401")),
|
||||
)
|
||||
.await;
|
||||
let r = reason(Some(&facts()), Some(&record()), Answers(Err("401"))).await;
|
||||
assert_eq!(r, Some(Reason::FetchFailed));
|
||||
}
|
||||
|
||||
|
|
@ -430,7 +318,7 @@ mod tests {
|
|||
let forged = Answers(Ok(Some(narinfo_signed_by(&SigningKey::from_bytes(
|
||||
&[9; 32],
|
||||
)))));
|
||||
let r = reason(Some(&facts()), Some(&record()), Some(REV), forged).await;
|
||||
let r = reason(Some(&facts()), Some(&record()), forged).await;
|
||||
assert_eq!(r, Some(Reason::BadSignature));
|
||||
}
|
||||
|
||||
|
|
@ -438,7 +326,6 @@ mod tests {
|
|||
fn the_labels_are_the_documented_ones() {
|
||||
let labels = [
|
||||
Reason::NoRecord,
|
||||
Reason::RevMismatch,
|
||||
Reason::NotCached,
|
||||
Reason::NoCredentials,
|
||||
Reason::FetchFailed,
|
||||
|
|
@ -449,7 +336,6 @@ mod tests {
|
|||
labels,
|
||||
[
|
||||
"no-record",
|
||||
"rev-mismatch",
|
||||
"not-cached",
|
||||
"no-credentials",
|
||||
"fetch-failed",
|
||||
|
|
|
|||
|
|
@ -42,12 +42,7 @@ pub struct ContainerView {
|
|||
/// up is indistinguishable from one an operator stopped on purpose.
|
||||
#[serde(default)]
|
||||
pub failed: bool,
|
||||
pub needs_update: bool,
|
||||
pub needs_login: bool,
|
||||
/// First 12 chars of the config commit the agent's last successful build
|
||||
/// pinned (`paths::applied_config_marker`).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deployed_sha: Option<String>,
|
||||
/// The Claude model the agent's harness is currently using, read from
|
||||
/// `state/hyperhive-harness.json["active_model"]`. `None` when the
|
||||
/// agent has never started a turn or the field is absent. Only
|
||||
|
|
@ -97,9 +92,7 @@ impl From<ContainerView> for hive_sh4re::container::AgentStatusRow {
|
|||
name: v.name,
|
||||
running: v.running,
|
||||
failed: v.failed,
|
||||
needs_update: v.needs_update,
|
||||
needs_login: v.needs_login,
|
||||
deployed_sha: v.deployed_sha,
|
||||
pending_reminders: 0,
|
||||
paused: v.paused,
|
||||
active_model: v.active_model,
|
||||
|
|
@ -153,11 +146,6 @@ pub async fn build_all() -> anyhow::Result<Vec<ContainerView>> {
|
|||
let Ok(logical) = hive_types::Ident::parse(logical) else {
|
||||
continue;
|
||||
};
|
||||
let deployed_full =
|
||||
std::fs::read_to_string(crate::paths::applied_config_marker(logical.as_str())).ok();
|
||||
let needs_update =
|
||||
crate::auto_update::agent_config_pending(logical.as_str(), deployed_full.as_deref());
|
||||
let deployed_sha = deployed_full.as_deref().map(short_rev);
|
||||
// One `systemctl` call for both facts: `unit_state` is the same
|
||||
// shell-out `is_running` makes, minus `--quiet`. Asking twice would
|
||||
// double the per-agent subprocess count on every SSE scan.
|
||||
|
|
@ -211,9 +199,7 @@ pub async fn build_all() -> anyhow::Result<Vec<ContainerView>> {
|
|||
failed,
|
||||
container: c.clone(),
|
||||
name: logical.into_string(),
|
||||
needs_update,
|
||||
needs_login,
|
||||
deployed_sha,
|
||||
active_model,
|
||||
status_text,
|
||||
status_set_at,
|
||||
|
|
@ -378,27 +364,36 @@ pub fn hive_swarm_names() -> (Option<String>, Option<String>) {
|
|||
(read("HYPERHIVE_HIVE_NAME"), read("HYPERHIVE_SWARM_NAME"))
|
||||
}
|
||||
|
||||
/// First 12 characters of a rev. Counts chars, not bytes, so a byte slice at 12
|
||||
/// cannot split a multi-byte char.
|
||||
fn short_rev(rev: &str) -> String {
|
||||
rev.chars().take(12).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::short_rev;
|
||||
|
||||
/// The row says nothing of an update being pending or of a deployed commit:
|
||||
/// the hive applies a changed declaration on its own, so there is no state
|
||||
/// for an operator to act on.
|
||||
#[test]
|
||||
fn short_rev_truncates_and_keeps_short() {
|
||||
assert_eq!(short_rev("0123456789abcdef"), "0123456789ab");
|
||||
assert_eq!(short_rev("abc123"), "abc123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_rev_survives_multibyte_at_boundary() {
|
||||
// 'é' is 2 bytes and straddles byte index 12.
|
||||
let rev = "0123456789aébcdef";
|
||||
assert_eq!(short_rev(rev), "0123456789aé");
|
||||
fn the_row_carries_no_update_state() {
|
||||
use super::ContainerView;
|
||||
let view = ContainerView {
|
||||
name: "alice".to_owned(),
|
||||
container: "h-alice".to_owned(),
|
||||
port: 7000,
|
||||
running: true,
|
||||
failed: false,
|
||||
needs_login: false,
|
||||
active_model: None,
|
||||
status_text: None,
|
||||
status_set_at: None,
|
||||
paused: false,
|
||||
};
|
||||
let value = serde_json::to_value(&view).unwrap();
|
||||
assert_eq!(value["running"], true, "control: the lookup finds fields");
|
||||
for gone in ["needs_update", "deployed_sha"] {
|
||||
assert!(value.get(gone).is_none(), "{gone} is still serialized");
|
||||
}
|
||||
let row = serde_json::to_value(hive_sh4re::container::AgentStatusRow::from(view)).unwrap();
|
||||
assert_eq!(row["running"], true);
|
||||
for gone in ["needs_update", "deployed_sha"] {
|
||||
assert!(row.get(gone).is_none(), "{gone} is still serialized");
|
||||
}
|
||||
}
|
||||
|
||||
/// The `From` impl both `handle_agent_status`'s poll path and
|
||||
|
|
@ -417,9 +412,7 @@ mod tests {
|
|||
port: 7000,
|
||||
running: true,
|
||||
failed: false,
|
||||
needs_update: true,
|
||||
needs_login: false,
|
||||
deployed_sha: Some("abc123def456".to_owned()),
|
||||
active_model: Some("claude-opus".to_owned()),
|
||||
status_text: Some("shipping".to_owned()),
|
||||
status_set_at: Some(Utc.timestamp_opt(1_700_000_000, 0).unwrap()),
|
||||
|
|
@ -430,9 +423,7 @@ mod tests {
|
|||
assert_eq!(row.name, "alice");
|
||||
assert!(row.running);
|
||||
assert!(!row.failed);
|
||||
assert!(row.needs_update);
|
||||
assert!(!row.needs_login);
|
||||
assert_eq!(row.deployed_sha.as_deref(), Some("abc123def456"));
|
||||
assert!(row.paused);
|
||||
assert_eq!(row.active_model.as_deref(), Some("claude-opus"));
|
||||
assert_eq!(row.status_text.as_deref(), Some("shipping"));
|
||||
|
|
|
|||
|
|
@ -507,7 +507,7 @@ impl Coordinator {
|
|||
/// and one `DashboardEvent::ContainerRemoved` per disappeared row.
|
||||
/// Call after any mutation that could affect what
|
||||
/// `nixos-container list` returns or what a row's
|
||||
/// `running` / `needs_update` / `needs_login` / `deployed_sha`
|
||||
/// `running` / `needs_login`
|
||||
/// resolves to — lifecycle ops, destroy,
|
||||
/// rebuild, and the crash-watcher's periodic poll.
|
||||
/// Cheap when nothing changed (one `nixos-container list` + a
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
//! Container lifecycle endpoints for the dashboard.
|
||||
//!
|
||||
//! Rebuild / restart / start / stop (hard + graceful) / update-all all
|
||||
//! Rebuild / restart / start / stop (hard + graceful) all
|
||||
//! insert DAGs into the job queue — the power ops via
|
||||
//! [`crate::job_queue::power`], the static shapes straight through
|
||||
//! `JobQueue::insert` — so each shows a visible queued→running transient on
|
||||
|
|
@ -311,34 +311,6 @@ fn set_paused_error_response(
|
|||
}
|
||||
}
|
||||
|
||||
/// Queue a rebuild DAG for every live agent
|
||||
/// container.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/update-all",
|
||||
responses(
|
||||
(status = 200, description = "rebuilds queued", body = String),
|
||||
(status = 500, description = "container list unreadable, nothing queued"),
|
||||
),
|
||||
tag = "lifecycle_ops"
|
||||
)]
|
||||
pub(super) async fn post_update_all(State(state): State<AppState>) -> Response {
|
||||
let agents = match lifecycle::agent_names(lifecycle::list().await) {
|
||||
Ok(agents) => agents,
|
||||
Err(e) => return error_response(&format!("update-all: listing containers: {e:#}")),
|
||||
};
|
||||
for logical in agents {
|
||||
if let Err(e) = state.coord.job_queue.insert_job(|b| {
|
||||
crate::job_queue::templates::rebuild(b, &logical);
|
||||
Vec::new()
|
||||
}) {
|
||||
tracing::error!(agent = %logical, error = ?e, "update-all: insert failed");
|
||||
}
|
||||
}
|
||||
state.coord.emit_rebuild_queue_snapshot();
|
||||
(StatusCode::OK, "ok").into_response()
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default, ToSchema)]
|
||||
pub(super) struct DestroyForm {
|
||||
#[serde(default)]
|
||||
|
|
|
|||
|
|
@ -119,7 +119,6 @@ fn app(coord: Arc<Coordinator>) -> axum::Router {
|
|||
.routes(routes!(lifecycle_ops::post_rebuild))
|
||||
.routes(routes!(lifecycle_ops::post_pause))
|
||||
.routes(routes!(lifecycle_ops::post_resume))
|
||||
.routes(routes!(lifecycle_ops::post_update_all))
|
||||
.routes(routes!(tombstones::post_purge_tombstone))
|
||||
.routes(routes!(build_logs::get_build_log_stream))
|
||||
.routes(routes!(state_snapshot::dashboard_stream))
|
||||
|
|
@ -369,7 +368,6 @@ mod router_build_probe {
|
|||
.routes(routes!(lifecycle_ops::post_rebuild))
|
||||
.routes(routes!(lifecycle_ops::post_pause))
|
||||
.routes(routes!(lifecycle_ops::post_resume))
|
||||
.routes(routes!(lifecycle_ops::post_update_all))
|
||||
.routes(routes!(tombstones::post_purge_tombstone))
|
||||
.routes(routes!(build_logs::get_build_log_stream))
|
||||
.routes(routes!(state_snapshot::dashboard_stream))
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ pub(super) struct StateSnapshot {
|
|||
/// can't be resolved. Feeds the dashboard start page so an operator
|
||||
/// can tell what build a hive is running without shelling in.
|
||||
hyperhive_rev: Option<String>,
|
||||
any_stale: bool,
|
||||
containers: Vec<ContainerView>,
|
||||
transients: Vec<TransientView>,
|
||||
/// State dirs (config history + claude creds + /state/ notes) that
|
||||
|
|
@ -176,7 +175,6 @@ pub(super) async fn api_state(
|
|||
);
|
||||
}
|
||||
let containers = state.coord.containers_snapshot().await;
|
||||
let any_stale = containers.iter().any(|c| c.needs_update);
|
||||
let transient_snapshot = state.coord.transient_snapshot();
|
||||
let transients = build_transient_views(&containers, &transient_snapshot);
|
||||
let tombstones = build_tombstone_views(&containers);
|
||||
|
|
@ -186,7 +184,6 @@ pub(super) async fn api_state(
|
|||
seq,
|
||||
hostname,
|
||||
hyperhive_rev: crate::auto_update::current_flake_rev(&state.coord.hyperhive_flake),
|
||||
any_stale,
|
||||
containers,
|
||||
transients,
|
||||
|
||||
|
|
@ -493,9 +490,7 @@ mod tests {
|
|||
port,
|
||||
running,
|
||||
failed: false,
|
||||
needs_update: false,
|
||||
needs_login: false,
|
||||
deployed_sha: None,
|
||||
active_model: None,
|
||||
status_text: None,
|
||||
status_set_at: None,
|
||||
|
|
|
|||
|
|
@ -71,8 +71,8 @@ pub enum DashboardEvent {
|
|||
transient_kind: String,
|
||||
},
|
||||
/// One container row changed — a new container appeared, an existing
|
||||
/// one flipped `running` / `needs_update` /
|
||||
/// `sha`, etc. Clients upsert by `container.name`. Payload carries
|
||||
/// one flipped `running` /
|
||||
/// `needs_login`, etc. Clients upsert by `container.name`. Payload carries
|
||||
/// the full row so cold-loaded clients and event-driven clients
|
||||
/// converge on the same render.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
//! This hive's reads of `internal/meta` and of an agent's config head, as
|
||||
//! [`crate::meta_rev::MetaSource`], under the hive's forge read token.
|
||||
|
||||
use forgejo_api::structs::RepoGetRawFileQuery;
|
||||
use forgejo_api::{ApiErrorKind, Forgejo, ForgejoError};
|
||||
use reqwest::StatusCode;
|
||||
|
||||
use super::reader::ReaderToken;
|
||||
use super::{CONFIG_ORG, bounded};
|
||||
use crate::meta_rev::MetaSource;
|
||||
|
||||
/// The org and repo the swarm commits its meta flake to.
|
||||
const META_OWNER: &str = "internal";
|
||||
const META_REPO: &str = "meta";
|
||||
|
||||
/// The branch an agent's config is deployed from, which `fetch_forge_main`
|
||||
/// and the swarm controller's pins both read.
|
||||
const CONFIG_BRANCH: &str = "main";
|
||||
|
||||
/// [`MetaSource`] over the forge API.
|
||||
pub(crate) struct ForgeSource {
|
||||
api: Forgejo,
|
||||
}
|
||||
|
||||
/// A source reading as this hive, or `None` when it holds no forge token.
|
||||
pub(crate) async fn internal_meta_source() -> Option<ForgeSource> {
|
||||
let token: ReaderToken = super::reader_token().await?;
|
||||
match token.api() {
|
||||
Ok(api) => Some(ForgeSource { api }),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "internal/meta: cannot build a forge client");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_not_found(e: &ForgejoError) -> bool {
|
||||
match e {
|
||||
ForgejoError::ApiError(api) => match api.error_kind() {
|
||||
ApiErrorKind::NotFound { .. } => true,
|
||||
ApiErrorKind::Other(s) => *s == StatusCode::NOT_FOUND,
|
||||
_ => false,
|
||||
},
|
||||
ForgejoError::UnexpectedStatusCode(s) => *s == StatusCode::NOT_FOUND,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl ForgeSource {
|
||||
/// The head commit of `branch` in `owner/repo`; `None` when the repo or
|
||||
/// the branch is not there.
|
||||
async fn branch_head(
|
||||
&self,
|
||||
owner: &str,
|
||||
repo: &str,
|
||||
branch: &str,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
let op = format!("read branch {branch} of {owner}/{repo}");
|
||||
match bounded(&op, self.api.repo_get_branch(owner, repo, branch)).await? {
|
||||
Ok(branch) => Ok(branch.commit.and_then(|c| c.id)),
|
||||
Err(e) if is_not_found(&e) => Ok(None),
|
||||
Err(e) => Err(anyhow::Error::new(e).context(op)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MetaSource for ForgeSource {
|
||||
async fn meta_head(&self) -> anyhow::Result<Option<String>> {
|
||||
use anyhow::Context as _;
|
||||
|
||||
let op = format!("read repo {META_OWNER}/{META_REPO}");
|
||||
let repo = match bounded(&op, self.api.repo_get(META_OWNER, META_REPO)).await? {
|
||||
Ok(repo) => repo,
|
||||
Err(e) if is_not_found(&e) => return Ok(None),
|
||||
Err(e) => return Err(anyhow::Error::new(e).context(op)),
|
||||
};
|
||||
if repo.empty.unwrap_or(false) {
|
||||
return Ok(None);
|
||||
}
|
||||
let branch = repo
|
||||
.default_branch
|
||||
.with_context(|| format!("{META_OWNER}/{META_REPO} names no default branch"))?;
|
||||
self.branch_head(META_OWNER, META_REPO, &branch).await
|
||||
}
|
||||
|
||||
async fn flake_at(&self, rev: &str) -> anyhow::Result<String> {
|
||||
let op = format!("read flake.nix of {META_OWNER}/{META_REPO} at {rev}");
|
||||
let query = RepoGetRawFileQuery {
|
||||
r#ref: Some(rev.to_owned()),
|
||||
};
|
||||
let raw = bounded(
|
||||
&op,
|
||||
self.api
|
||||
.repo_get_raw_file(META_OWNER, META_REPO, "flake.nix", query),
|
||||
)
|
||||
.await?
|
||||
.map_err(|e| anyhow::Error::new(e).context(op))?;
|
||||
Ok(String::from_utf8_lossy(&raw).into_owned())
|
||||
}
|
||||
|
||||
async fn config_head(&self, agent: &str) -> anyhow::Result<Option<String>> {
|
||||
self.branch_head(CONFIG_ORG, agent, CONFIG_BRANCH).await
|
||||
}
|
||||
}
|
||||
|
|
@ -4,18 +4,13 @@
|
|||
//! `internal/docs`, are swarm-controller's. Full design:
|
||||
//! `docs/integrations/forge.md`.
|
||||
|
||||
mod internal_meta;
|
||||
mod reader;
|
||||
|
||||
pub(crate) use internal_meta::internal_meta_source;
|
||||
pub use reader::reader_token;
|
||||
|
||||
use std::future::IntoFuture;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use forgejo_api::{Auth, Forgejo};
|
||||
use url::Url;
|
||||
|
||||
const FORGE_CONTAINER: &str = "hive-forge";
|
||||
|
|
@ -88,15 +83,6 @@ fn basic_auth_header(login: &str, token: &str) -> String {
|
|||
format!("Authorization: Basic {basic}")
|
||||
}
|
||||
|
||||
/// Forgejo org grouping every agent's config repo. Each agent is a **write collaborator
|
||||
/// on its own** `agent-configs/<name>` repo — the editable PR surface it
|
||||
/// pushes config-change branches to. `main`'s branch protection is
|
||||
/// swarm-controller's: merge and approval allowlisted to the `operators` team,
|
||||
/// so the agent can't push `main` or self-merge, and an operator's merge
|
||||
/// deploys through swarm-controller. The repos remain private, so an agent
|
||||
/// still can't reach *another* agent's config. hive-c0re only reads from it.
|
||||
pub(crate) const CONFIG_ORG: &str = "agent-configs";
|
||||
|
||||
/// Probe whether `hive-forge` exists as a nixos-container. Cheap —
|
||||
/// `nixos-container list` is just a directory scan in /etc. Routed
|
||||
/// through hive-priv: `nixos-container` needs root, and hive-c0re runs
|
||||
|
|
@ -108,70 +94,9 @@ pub async fn is_present() -> bool {
|
|||
stdout.lines().any(|l| l.trim() == FORGE_CONTAINER)
|
||||
}
|
||||
|
||||
/// Typed Forgejo API client for the local forge ([`forge_http_base()`]),
|
||||
/// authenticated as `token`. All Forgejo API calls go through clients built
|
||||
/// here — one place for the base URL and auth. The token is passed per call;
|
||||
/// the base URL is parsed once. Failures surface as
|
||||
/// `forgejo_api::ForgejoError`, whose Display carries the HTTP status
|
||||
/// and the API's error message (e.g. the validation reason on a 422).
|
||||
pub(crate) fn api(token: &str) -> Result<Forgejo> {
|
||||
static URL: OnceLock<Url> = OnceLock::new();
|
||||
let url = URL
|
||||
.get_or_init(|| Url::parse(forge_http_base()).expect("forge_http_base() is a valid URL"))
|
||||
.clone();
|
||||
Forgejo::new(Auth::Token(token), url).context("build forgejo api client")
|
||||
}
|
||||
|
||||
/// Upper bound on one Forgejo API request. `forgejo-api` takes no
|
||||
/// caller-built client, so the request itself has no timeout; every request
|
||||
/// in this module is wrapped in [`bounded`].
|
||||
const API_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Await one Forgejo API request for at most [`API_TIMEOUT`].
|
||||
///
|
||||
/// The outer `Err` is the deadline, naming `op`; the inner `Result` is the
|
||||
/// request's own outcome, left to the caller so it can still match on the
|
||||
/// error (`409 already exists`, `401`, ...).
|
||||
async fn bounded<F, T, E>(op: &str, request: F) -> Result<std::result::Result<T, E>>
|
||||
where
|
||||
F: IntoFuture<Output = std::result::Result<T, E>>,
|
||||
{
|
||||
tokio::time::timeout(API_TIMEOUT, request)
|
||||
.await
|
||||
.map_err(|_| anyhow::anyhow!("{op}: no answer from the forge within {API_TIMEOUT:?}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{API_TIMEOUT, bounded, git_url_with_base};
|
||||
|
||||
/// A forge that never answers surfaces as an error naming the operation,
|
||||
/// at exactly [`API_TIMEOUT`] — not as a hang.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_request_the_forge_never_answers_errors_at_the_deadline() {
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = bounded(
|
||||
"pending request",
|
||||
std::future::pending::<Result<(), forgejo_api::ForgejoError>>(),
|
||||
)
|
||||
.await
|
||||
.expect_err("a pending request must hit the deadline");
|
||||
assert_eq!(started.elapsed(), API_TIMEOUT);
|
||||
assert!(err.to_string().contains("pending request"), "{err}");
|
||||
}
|
||||
|
||||
/// The request's own result passes through for the caller to classify.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn an_answered_request_keeps_its_own_result() {
|
||||
let ok = bounded("op", async { Ok::<_, ()>(7) })
|
||||
.await
|
||||
.expect("in time");
|
||||
assert_eq!(ok, Ok(7));
|
||||
let failed = bounded("op", async { Err::<(), _>("409") })
|
||||
.await
|
||||
.expect("in time");
|
||||
assert_eq!(failed, Err("409"));
|
||||
}
|
||||
use super::git_url_with_base;
|
||||
|
||||
/// The remote carries **no credential**. `argv` is world-readable through
|
||||
/// `/proc/<pid>/cmdline`, so a token spliced in here would be published to
|
||||
|
|
|
|||
|
|
@ -26,11 +26,6 @@ impl ReaderToken {
|
|||
pub fn auth_header(&self) -> String {
|
||||
basic_auth_header(&self.login, &self.value)
|
||||
}
|
||||
|
||||
/// A forge API client acting as this token's account.
|
||||
pub(crate) fn api(&self) -> anyhow::Result<forgejo_api::Forgejo> {
|
||||
super::api(&self.value)
|
||||
}
|
||||
}
|
||||
|
||||
/// The token to read the forge with, or `None` when the store holds none for
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
//! Node executors — one async fn per [`NodeKind`], each a thin wrapper
|
||||
//! over existing `lifecycle.rs` / `meta_rev.rs` / `actions.rs` code. Node
|
||||
//! over existing `lifecycle.rs` / `actions.rs` code. Node
|
||||
//! executors keep their own internal error handling where it exists
|
||||
//! today (cold-start fallback inside `Reconcile`, warn-only forge sync in
|
||||
//! the `Swap` tail); DAG-level failure handling is cancel-downstream in
|
||||
|
|
@ -15,7 +15,6 @@ use hive_jobq::NodeId;
|
|||
|
||||
use super::model::NodeKind;
|
||||
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
|
||||
use crate::meta_rev::MetaBuild;
|
||||
use crate::power::{ReconcileAction, reconcile_action};
|
||||
use crate::stats::sweep_health::{self, SweepHealth};
|
||||
|
||||
|
|
@ -35,21 +34,17 @@ pub const GRACEFUL_STOP_TIMEOUT: std::time::Duration = std::time::Duration::from
|
|||
const PAUSE_ACK_TIMEOUT: std::time::Duration = GRACEFUL_STOP_TIMEOUT;
|
||||
|
||||
/// The container-state calls the executors below make on the host: systemd,
|
||||
/// `hive-priv`, the swarm's closure record, the `internal/meta` record. Production uses
|
||||
/// [`RealHost`], which forwards to `lifecycle` / `meta_rev` / `Coordinator`
|
||||
/// `hive-priv`, the swarm's closure record and the marker of the closure applied.
|
||||
/// Production uses [`RealHost`], which forwards to `lifecycle` / `Coordinator`
|
||||
/// unchanged; tests substitute a recording fake.
|
||||
pub(super) trait Host: Sync {
|
||||
/// Whether the agent's container unit is active.
|
||||
async fn is_running(&self, name: &str) -> bool;
|
||||
/// Stop the agent's container.
|
||||
async fn kill(&self, name: &str) -> Result<()>;
|
||||
/// The `internal/meta` commit `MetaSync` recorded for the agent.
|
||||
fn recorded_rev(&self, name: &str) -> Result<String>;
|
||||
/// The record the swarm's builder wrote for the agent; `None` when it wrote
|
||||
/// none.
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>>;
|
||||
/// The pins `internal/meta` at the commit `rev` holds for the agent.
|
||||
async fn pins_at(&self, name: &str, rev: &str) -> Result<MetaBuild>;
|
||||
/// Download `store_path` from the swarm's cache into the store, never
|
||||
/// building it.
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()>;
|
||||
|
|
@ -72,15 +67,8 @@ pub(super) trait Host: Sync {
|
|||
async fn prepare_rebuild_dirs(&self, name: &str, paths: &AgentPaths) -> Result<()>;
|
||||
/// Rewrite the host-side files that follow the agent set.
|
||||
async fn refresh_agent_set(&self) -> Result<()>;
|
||||
/// Pin the `internal/meta` commit the agent builds from.
|
||||
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild>;
|
||||
/// Record `build` as the commit `Prebuild` and `Swap` expect the swarm's
|
||||
/// closure to be built from.
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// The build `MetaSync` recorded, if any.
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild>;
|
||||
/// Mark the agent as running the pins of `build`.
|
||||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()>;
|
||||
/// Mark the agent as running the closure at `store_path`.
|
||||
fn mark_deployed(&self, name: &str, store_path: &str) -> Result<()>;
|
||||
/// First-spawn provisioning: repos, state subvolume, agent-set refresh.
|
||||
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()>;
|
||||
/// `nixos-container create` from the closure `toplevel`.
|
||||
|
|
@ -117,18 +105,10 @@ impl Host for RealHost {
|
|||
crate::lifecycle::kill(name).await
|
||||
}
|
||||
|
||||
fn recorded_rev(&self, name: &str) -> Result<String> {
|
||||
Ok(crate::meta_rev::recorded_or_err(name)?.rev)
|
||||
}
|
||||
|
||||
async fn closure_record(&self, name: &str) -> Result<Option<AgentClosure>> {
|
||||
crate::attic::closure::record(name).await
|
||||
}
|
||||
|
||||
async fn pins_at(&self, name: &str, rev: &str) -> Result<MetaBuild> {
|
||||
crate::meta_rev::resolve_at(name, rev).await
|
||||
}
|
||||
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> Result<()> {
|
||||
crate::priv_client::realise_closure(name, store_path).await
|
||||
}
|
||||
|
|
@ -165,20 +145,8 @@ impl Host for RealHost {
|
|||
crate::agent_set::refresh(&agents).await
|
||||
}
|
||||
|
||||
async fn resolve_meta(&self, name: &str) -> Result<MetaBuild> {
|
||||
crate::meta_rev::resolve(name).await
|
||||
}
|
||||
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> Result<()> {
|
||||
crate::meta_rev::record(name, build)
|
||||
}
|
||||
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild> {
|
||||
crate::meta_rev::recorded(name)
|
||||
}
|
||||
|
||||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> Result<()> {
|
||||
crate::meta_rev::mark_deployed(name, build)
|
||||
fn mark_deployed(&self, name: &str, store_path: &str) -> Result<()> {
|
||||
crate::attic::closure::mark_deployed(name, store_path)
|
||||
}
|
||||
|
||||
async fn provision(&self, name: &str, paths: &AgentPaths) -> Result<()> {
|
||||
|
|
@ -283,7 +251,7 @@ async fn dispatch(
|
|||
NodeKind::MetaSync { .. } => run_meta_sync(host, agent).await,
|
||||
NodeKind::Prebuild { .. } => run_prebuild(host, agent).await,
|
||||
NodeKind::Swap { .. } => run_swap(host, coord, agent, id).await,
|
||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(host, coord, agent).await,
|
||||
NodeKind::RebuildBookkeeping { .. } => run_rebuild_bookkeeping(coord).await,
|
||||
NodeKind::Provision { .. } => run_provision(host, agent).await,
|
||||
NodeKind::Create { .. } => run_create(host, agent).await,
|
||||
NodeKind::DestroyContainer { .. } => run_destroy_container(host, coord, agent).await,
|
||||
|
|
@ -502,12 +470,9 @@ fn run_set_wanted(coord: &Arc<Coordinator>, agent: &str, up: bool) -> Result<()>
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// The rebuild's preamble: runtime-dir prep, the agent-set refresh, and the
|
||||
/// `internal/meta` commit `Prebuild` and `Swap` expect the swarm's closure to
|
||||
/// be built from.
|
||||
///
|
||||
/// Fails when `internal/meta` does not pin the agent's current config yet:
|
||||
/// there is no commit to deploy.
|
||||
/// The rebuild's preamble: runtime-dir prep, the archetype and agent-set
|
||||
/// refresh. Reads nothing from the forge: what a rebuild deploys is the swarm's
|
||||
/// declared closure, read by `Prebuild` and `Swap`.
|
||||
async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
||||
// Runs while the agent is still up — the runtime dir and MCP listener
|
||||
// already exist. Use the pure path accessor; no need to re-register the
|
||||
|
|
@ -521,9 +486,6 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
|||
}
|
||||
host.prepare_rebuild_dirs(name, &paths).await?;
|
||||
host.refresh_agent_set().await?;
|
||||
// Resolved once here so `Prebuild` and `Swap` expect the same commit.
|
||||
let build = host.resolve_meta(name).await?;
|
||||
host.record_meta(name, &build)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -534,9 +496,8 @@ async fn run_meta_sync(host: &impl Host, name: &str) -> Result<()> {
|
|||
/// is already down: there is no uptime to protect, and `Swap` fails the same way.
|
||||
async fn run_prebuild(host: &impl Host, name: &str) -> Result<()> {
|
||||
if host.is_running(name).await {
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let closure = closure_for(host, name, &rev, "prebuild").await?;
|
||||
tracing::info!(%name, %rev, "prebuild: substituted {}", closure.path);
|
||||
let closure = closure_for(host, name, "prebuild").await?;
|
||||
tracing::info!(%name, "prebuild: substituted {closure}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -550,32 +511,19 @@ enum NoClosure {
|
|||
Unpullable(String),
|
||||
}
|
||||
|
||||
/// The closure the swarm built for an agent, in the store.
|
||||
struct Closure {
|
||||
path: String,
|
||||
/// The `internal/meta` commit the builder built it from.
|
||||
meta_rev: String,
|
||||
}
|
||||
|
||||
/// The closure the swarm built for `name`, downloaded into the store. The
|
||||
/// deploy applies the swarm's closure whatever commit it was built from: when
|
||||
/// that is not the `rev` this hive chose, it is logged, and the caller records
|
||||
/// the pins of the commit actually deployed ([`deployed_pins`]).
|
||||
/// The store path of the closure the swarm declares for `name`, downloaded into
|
||||
/// the store. Whatever `internal/meta` commit it was built from, it is the one
|
||||
/// deployed.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`without_closure`]'s refusal, after logging it.
|
||||
async fn closure_for(host: &impl Host, name: &str, rev: &str, step: &str) -> Result<Closure> {
|
||||
pull_closure(host, name, rev, step)
|
||||
async fn closure_for(host: &impl Host, name: &str, step: &str) -> Result<String> {
|
||||
pull_closure(host, name)
|
||||
.await
|
||||
.map_err(|why| without_closure(name, rev, step, &why))
|
||||
.map_err(|why| without_closure(name, step, &why))
|
||||
}
|
||||
|
||||
async fn pull_closure(
|
||||
host: &impl Host,
|
||||
name: &str,
|
||||
rev: &str,
|
||||
step: &str,
|
||||
) -> Result<Closure, NoClosure> {
|
||||
async fn pull_closure(host: &impl Host, name: &str) -> Result<String, NoClosure> {
|
||||
let record = host
|
||||
.closure_record(name)
|
||||
.await
|
||||
|
|
@ -589,52 +537,7 @@ async fn pull_closure(
|
|||
host.realise_closure(name, &record.store_path)
|
||||
.await
|
||||
.map_err(|e| NoClosure::Unpullable(format!("{e:#}")))?;
|
||||
if record.meta_rev != rev && !same_pins(host, name, &record.meta_rev, rev).await {
|
||||
tracing::warn!(
|
||||
%name, %rev, built_from = %record.meta_rev,
|
||||
"{step}: the swarm built {} from internal/meta {}, not the {rev} this hive chose; deploying the swarm's",
|
||||
record.store_path, record.meta_rev
|
||||
);
|
||||
}
|
||||
Ok(Closure {
|
||||
path: record.store_path,
|
||||
meta_rev: record.meta_rev,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the agent has the same pins at the commits `built` and `chosen`. A
|
||||
/// commit whose pins cannot be read counts as different.
|
||||
async fn same_pins(host: &impl Host, name: &str, built: &str, chosen: &str) -> bool {
|
||||
match (
|
||||
host.pins_at(name, built).await,
|
||||
host.pins_at(name, chosen).await,
|
||||
) {
|
||||
(Ok(built), Ok(chosen)) => built.same_pins(&chosen),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The pins to record for a deploy of `closure` when this hive chose the commit
|
||||
/// `rev`: those of the commit the closure was built from, so the markers say
|
||||
/// what runs. `None` when they are the same commit.
|
||||
///
|
||||
/// # Errors
|
||||
/// The pins of the closure's commit could not be read. The caller fails before
|
||||
/// applying anything.
|
||||
async fn deployed_pins(
|
||||
host: &impl Host,
|
||||
name: &str,
|
||||
closure: &Closure,
|
||||
rev: &str,
|
||||
) -> Result<Option<MetaBuild>> {
|
||||
if closure.meta_rev == rev {
|
||||
return Ok(None);
|
||||
}
|
||||
let pins = host
|
||||
.pins_at(name, &closure.meta_rev)
|
||||
.await
|
||||
.with_context(|| format!("{name}: reading the pins of the commit the swarm built from"))?;
|
||||
Ok(Some(pins))
|
||||
Ok(record.store_path)
|
||||
}
|
||||
|
||||
/// What a rebuild or create does when there is no closure to deploy: it
|
||||
|
|
@ -645,18 +548,19 @@ async fn deployed_pins(
|
|||
/// This is the one place that decides what an unpullable closure (the cache
|
||||
/// unreachable, a TLS error, a failed download) costs. A rebuild never builds
|
||||
/// locally on the hive.
|
||||
fn without_closure(name: &str, rev: &str, step: &str, why: &NoClosure) -> anyhow::Error {
|
||||
fn without_closure(name: &str, step: &str, why: &NoClosure) -> anyhow::Error {
|
||||
let reason = match why {
|
||||
NoClosure::NoRecord => "the swarm has recorded no build of this agent".to_owned(),
|
||||
NoClosure::Unpullable(e) => format!("the swarm's closure cannot be pulled: {e}"),
|
||||
};
|
||||
tracing::error!(%name, %rev, "{step}: {reason}; nothing was changed");
|
||||
tracing::error!(%name, "{step}: {reason}; nothing was changed");
|
||||
anyhow::anyhow!("{name}: {reason}")
|
||||
}
|
||||
|
||||
/// Profile-swap: re-apply drop-ins (rebuild is the reconcile verb), point the
|
||||
/// container at the swarm's closure, then the post-rebuild bookkeeping tail
|
||||
/// (rev marker, `Rebuilt` event, forge sync, kick, rescan).
|
||||
/// container at the swarm's closure and, once that holds, mark the agent as
|
||||
/// running it. The post-rebuild bookkeeping tail (`Rebuilt` event, forge sync,
|
||||
/// kick, rescan) is the next node.
|
||||
/// The recovery-start on failure is NOT here — the DAG's tail
|
||||
/// `Reconcile` runs after this node terminal ok *or* fail.
|
||||
async fn run_swap(
|
||||
|
|
@ -670,21 +574,20 @@ async fn run_swap(
|
|||
let agent_dir = crate::paths::agent_runtime_dir(name);
|
||||
let hive = coord.hive_env();
|
||||
let paths = Coordinator::agent_paths(name, agent_dir)?;
|
||||
let rev = host.recorded_rev(name)?;
|
||||
let closure = closure_for(host, name, &rev, "swap").await?;
|
||||
let deployed = deployed_pins(host, name, &closure, &rev).await?;
|
||||
tracing::info!(%name, %rev, "swap: applying {}", closure.path);
|
||||
let closure = closure_for(host, name, "swap").await?;
|
||||
tracing::info!(%name, "swap: applying {closure}");
|
||||
let result = host
|
||||
.swap_update(name, &hive, &paths, &closure.path, id.get())
|
||||
.swap_update(name, &hive, &paths, &closure, id.get())
|
||||
.await;
|
||||
// The bookkeeping node marks what the record holds, so it holds what was
|
||||
// applied.
|
||||
// The marker names what was applied; a failed write is not a failed swap
|
||||
// (the container runs the closure), it only means the next comparison
|
||||
// sees the declaration as new.
|
||||
if result.is_ok()
|
||||
&& let Some(deployed) = &deployed
|
||||
&& let Err(e) = host.mark_deployed(name, &closure)
|
||||
{
|
||||
host.record_meta(name, deployed)?;
|
||||
tracing::warn!(%name, error = ?e, "write the deployed-closure marker failed");
|
||||
}
|
||||
// On success the Ok-only bookkeeping tail (rev marker, forge
|
||||
// On success the Ok-only bookkeeping tail (forge
|
||||
// sync, kick, rescan, snapshot) runs in the sibling `RebuildBookkeeping` node,
|
||||
// which deps `AfterOk(Swap)`. On failure `RebuildBookkeeping` is cancel-cascaded
|
||||
// and the tail `Reconcile` (`AfterAny(RebuildBookkeeping)`) handles recovery; here
|
||||
|
|
@ -702,16 +605,7 @@ async fn run_swap(
|
|||
/// means the profile swap succeeded. Store/forge work only — no nix
|
||||
/// build (build-slot-exempt); the agent lease taken at `Swap` is still held
|
||||
/// (the whole chain up to `Reconcile` is one agent's subgraph).
|
||||
async fn run_rebuild_bookkeeping(
|
||||
host: &impl Host,
|
||||
coord: &Arc<Coordinator>,
|
||||
name: &str,
|
||||
) -> Result<()> {
|
||||
if let Some(build) = host.recorded_build(name)
|
||||
&& let Err(e) = host.mark_deployed(name, &build)
|
||||
{
|
||||
tracing::warn!(%name, error = ?e, "write rev markers failed");
|
||||
}
|
||||
async fn run_rebuild_bookkeeping(coord: &Arc<Coordinator>) -> Result<()> {
|
||||
// The `Rebuilt` manager event is emitted exactly once per agent by the DAG's
|
||||
// `EmitRebuilt` tail — emitting ok here and letting a failed tail `Reconcile`
|
||||
// add a contradictory !ok would double-report the same rebuild.
|
||||
|
|
@ -724,7 +618,7 @@ async fn run_rebuild_bookkeeping(
|
|||
// alongside `run_start`'s own kick for the same boot. `hive-agent` now
|
||||
// self-announces once on its own startup instead — one message per
|
||||
// actual boot, not one per DAG node that happens to run during it.
|
||||
// Rescan so dashboards drop the "needs update" chip.
|
||||
// Rescan so dashboards see the rebuilt container.
|
||||
coord.rescan_containers_and_emit().await;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -743,19 +637,15 @@ async fn run_provision(host: &impl Host, name: &str) -> Result<()> {
|
|||
/// dir creation and MCP listener registration are deferred to the tail
|
||||
/// `Reconcile` (`converge_start_preamble` + `register_agent`) so this
|
||||
/// node stays purely "create", not "create + start". Marks the agent as
|
||||
/// running the pins it was created from.
|
||||
/// running the closure it was created from.
|
||||
///
|
||||
/// The container is created from the swarm's closure for the resolved commit;
|
||||
/// with none to deploy the create fails and makes no container.
|
||||
/// The container is created from the swarm's declared closure; with none to
|
||||
/// deploy the create fails and makes no container.
|
||||
async fn run_create(host: &impl Host, name: &str) -> Result<()> {
|
||||
let build = host.resolve_meta(name).await?;
|
||||
let closure = closure_for(host, name, &build.rev, "create").await?;
|
||||
let build = deployed_pins(host, name, &closure, &build.rev)
|
||||
.await?
|
||||
.unwrap_or(build);
|
||||
tracing::info!(%name, rev = %build.rev, "create: substituted {}", closure.path);
|
||||
host.create_only(name, &closure.path).await?;
|
||||
host.mark_deployed(name, &build)?;
|
||||
let closure = closure_for(host, name, "create").await?;
|
||||
tracing::info!(%name, "create: substituted {closure}");
|
||||
host.create_only(name, &closure).await?;
|
||||
host.mark_deployed(name, &closure)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ use super::{
|
|||
};
|
||||
use crate::coordinator::{AgentPaths, Coordinator, HiveEnv};
|
||||
use crate::job_queue::model::NodeKind;
|
||||
use crate::meta_rev::MetaBuild;
|
||||
use crate::power::Wanted;
|
||||
use swarm_wire::agent_closures::AgentClosure;
|
||||
|
||||
|
|
@ -30,19 +29,16 @@ const CLOSURE: &str = "/nix/store/0123456789abcdfghijklmnpqrsvwxyz-nixos-system-
|
|||
#[derive(Default)]
|
||||
struct FakeHost {
|
||||
running: bool,
|
||||
rev: Option<String>,
|
||||
/// What the swarm's builder recorded for the agent; `None` reads as no
|
||||
/// record.
|
||||
built: Option<AgentClosure>,
|
||||
/// Store paths a download put in the store.
|
||||
realised: Mutex<Vec<String>>,
|
||||
/// Names of the calls that fail (`kill`, `realise`, `closure_record`, `swap_update`,
|
||||
/// `set_paused`, ...). `recorded_build` fails by finding nothing.
|
||||
/// `set_paused`, `mark_deployed`, ...).
|
||||
failing: Vec<&'static str>,
|
||||
/// The sweeps and pulls never finish, so only a deadline ends them.
|
||||
hang: bool,
|
||||
/// The agent has the same pins at every commit.
|
||||
unmoved: bool,
|
||||
runtime: PathBuf,
|
||||
purge: Vec<PathBuf>,
|
||||
calls: Mutex<Vec<String>>,
|
||||
|
|
@ -52,8 +48,7 @@ impl FakeHost {
|
|||
fn up() -> Self {
|
||||
Self {
|
||||
running: true,
|
||||
rev: Some(REV.to_owned()),
|
||||
built: Some(closure(REV, true)),
|
||||
built: Some(closure(true)),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
|
@ -61,20 +56,14 @@ impl FakeHost {
|
|||
fn down() -> Self {
|
||||
Self {
|
||||
running: false,
|
||||
rev: Some(REV.to_owned()),
|
||||
built: Some(closure(REV, true)),
|
||||
built: Some(closure(true)),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// The builder recorded `meta_rev` for the agent, pushed or not.
|
||||
fn built(mut self, meta_rev: &str, cached: bool) -> Self {
|
||||
self.built = Some(closure(meta_rev, cached));
|
||||
self
|
||||
}
|
||||
|
||||
fn unmoved(mut self) -> Self {
|
||||
self.unmoved = true;
|
||||
/// The builder recorded the closure for the agent, pushed or not.
|
||||
fn built(mut self, cached: bool) -> Self {
|
||||
self.built = Some(closure(cached));
|
||||
self
|
||||
}
|
||||
|
||||
|
|
@ -124,31 +113,11 @@ impl Host for FakeHost {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn recorded_rev(&self, name: &str) -> anyhow::Result<String> {
|
||||
self.record(format!("recorded_rev {name}"));
|
||||
self.rev
|
||||
.clone()
|
||||
.ok_or_else(|| anyhow::anyhow!("{name}: no internal/meta commit was recorded"))
|
||||
}
|
||||
|
||||
async fn closure_record(&self, name: &str) -> anyhow::Result<Option<AgentClosure>> {
|
||||
self.step("closure_record", format!("closure_record {name}"))?;
|
||||
Ok(self.built.clone())
|
||||
}
|
||||
|
||||
async fn pins_at(&self, name: &str, rev: &str) -> anyhow::Result<MetaBuild> {
|
||||
self.step("pins_at", format!("pins_at {name} {rev}"))?;
|
||||
Ok(MetaBuild {
|
||||
rev: rev.to_owned(),
|
||||
hyperhive: "dh".to_owned(),
|
||||
config: Some(if self.unmoved {
|
||||
"dc".to_owned()
|
||||
} else {
|
||||
format!("dc-{rev}")
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
async fn realise_closure(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
|
||||
self.record(format!("realise {name} {store_path}"));
|
||||
if self.fails("realise") {
|
||||
|
|
@ -200,24 +169,10 @@ impl Host for FakeHost {
|
|||
self.step("refresh_agent_set", "refresh_agent_set".to_owned())
|
||||
}
|
||||
|
||||
async fn resolve_meta(&self, name: &str) -> anyhow::Result<MetaBuild> {
|
||||
self.step("resolve_meta", format!("resolve_meta {name}"))?;
|
||||
Ok(build())
|
||||
}
|
||||
|
||||
fn record_meta(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
|
||||
self.step("record_meta", format!("record_meta {name} {}", build.rev))
|
||||
}
|
||||
|
||||
fn recorded_build(&self, name: &str) -> Option<MetaBuild> {
|
||||
self.record(format!("recorded_build {name}"));
|
||||
(!self.fails("recorded_build")).then(build)
|
||||
}
|
||||
|
||||
fn mark_deployed(&self, name: &str, build: &MetaBuild) -> anyhow::Result<()> {
|
||||
fn mark_deployed(&self, name: &str, store_path: &str) -> anyhow::Result<()> {
|
||||
self.step(
|
||||
"mark_deployed",
|
||||
format!("mark_deployed {name} {}", build.rev),
|
||||
format!("mark_deployed {name} {store_path}"),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -277,10 +232,10 @@ impl Host for FakeHost {
|
|||
}
|
||||
}
|
||||
|
||||
/// The builder's record of a closure built from `internal/meta` at `meta_rev`.
|
||||
fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
|
||||
/// The builder's record of a closure.
|
||||
fn closure(cached: bool) -> AgentClosure {
|
||||
AgentClosure {
|
||||
meta_rev: meta_rev.to_owned(),
|
||||
meta_rev: REV.to_owned(),
|
||||
store_path: CLOSURE.to_owned(),
|
||||
system: "x86_64-linux".to_owned(),
|
||||
built_unix: 1,
|
||||
|
|
@ -289,14 +244,6 @@ fn closure(meta_rev: &str, cached: bool) -> AgentClosure {
|
|||
}
|
||||
}
|
||||
|
||||
fn build() -> MetaBuild {
|
||||
MetaBuild {
|
||||
rev: REV.to_owned(),
|
||||
hyperhive: "h".to_owned(),
|
||||
config: Some("c".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn coordinator() -> (tempfile::TempDir, Arc<Coordinator>) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let coord = Coordinator::open(
|
||||
|
|
@ -325,8 +272,6 @@ fn node_id() -> NodeId {
|
|||
|
||||
// --- Prebuild ---------------------------------------------------------------
|
||||
|
||||
const OTHER_REV: &str = "3333333333333333333333333333333333333333";
|
||||
|
||||
/// What a block logged, level and text: the message and each field.
|
||||
#[derive(Clone, Default)]
|
||||
struct Logged(Arc<Mutex<Vec<(tracing::Level, String)>>>);
|
||||
|
|
@ -384,11 +329,7 @@ async fn logging<T>(fut: impl std::future::Future<Output = T>) -> (T, Logged) {
|
|||
fn without_a_closure() -> [(&'static str, FakeHost, &'static str); 4] {
|
||||
[
|
||||
("no record", FakeHost::up().without_record(), "no build"),
|
||||
(
|
||||
"not pushed",
|
||||
FakeHost::up().built(REV, false),
|
||||
"did not push",
|
||||
),
|
||||
("not pushed", FakeHost::up().built(false), "did not push"),
|
||||
(
|
||||
"download fails",
|
||||
FakeHost::up().failing("realise"),
|
||||
|
|
@ -411,7 +352,6 @@ async fn prebuild_downloads_the_swarms_closure_when_the_container_is_up() {
|
|||
host.calls(),
|
||||
[
|
||||
"is_running alice".to_owned(),
|
||||
"recorded_rev alice".to_owned(),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
]
|
||||
|
|
@ -454,39 +394,6 @@ async fn a_rebuild_without_a_closure_fails_the_prebuild_and_logs_an_error() {
|
|||
}
|
||||
}
|
||||
|
||||
/// Another commit that pins the agent the same is nothing to warn about.
|
||||
#[tokio::test]
|
||||
async fn a_closure_built_from_a_commit_with_the_same_pins_is_deployed_quietly() {
|
||||
let host = FakeHost::up().built(OTHER_REV, true).unmoved();
|
||||
let (result, log) = logging(run_prebuild(&host, "alice")).await;
|
||||
result.expect("drift is not a refusal");
|
||||
assert!(host.calls().contains(&format!("realise alice {CLOSURE}")));
|
||||
assert!(log.at(tracing::Level::WARN).is_empty());
|
||||
}
|
||||
|
||||
/// The builder's record is not for the commit this hive chose: its closure is
|
||||
/// deployed and the difference is a warning.
|
||||
#[tokio::test]
|
||||
async fn a_closure_built_from_another_commit_is_deployed_with_a_warning() {
|
||||
let host = FakeHost::up().built(OTHER_REV, true);
|
||||
let (result, log) = logging(run_prebuild(&host, "alice")).await;
|
||||
result.expect("drift is not a refusal");
|
||||
assert!(
|
||||
host.calls().contains(&format!("realise alice {CLOSURE}")),
|
||||
"{:?}",
|
||||
host.calls()
|
||||
);
|
||||
let warnings = log.at(tracing::Level::WARN);
|
||||
assert_eq!(warnings.len(), 1, "{warnings:?}");
|
||||
assert!(
|
||||
[REV, OTHER_REV, CLOSURE]
|
||||
.iter()
|
||||
.all(|part| warnings[0].contains(part)),
|
||||
"{warnings:?}"
|
||||
);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_stopped_container_looks_up_nothing() {
|
||||
let host = FakeHost::down();
|
||||
|
|
@ -494,22 +401,6 @@ async fn a_stopped_container_looks_up_nothing() {
|
|||
assert_eq!(host.calls(), ["is_running alice"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prebuild_without_a_recorded_rev_fails_before_looking_up_a_closure() {
|
||||
let host = FakeHost {
|
||||
rev: None,
|
||||
..FakeHost::up()
|
||||
};
|
||||
let err = run_prebuild(&host, "alice")
|
||||
.await
|
||||
.expect_err("nothing to deploy");
|
||||
assert!(
|
||||
format!("{err:#}").contains("no internal/meta commit"),
|
||||
"{err:#}"
|
||||
);
|
||||
assert!(!host.calls().iter().any(|c| c.starts_with("closure_record")));
|
||||
}
|
||||
|
||||
// --- Swap -------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -521,10 +412,10 @@ async fn swap_applies_the_swarms_closure() {
|
|||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"recorded_rev alice".to_owned(),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("swap_update alice {CLOSURE} node={}", id.get()),
|
||||
format!("mark_deployed alice {CLOSURE}"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
|
@ -551,53 +442,40 @@ async fn a_swap_without_a_closure_never_updates_and_logs_an_error() {
|
|||
}
|
||||
}
|
||||
|
||||
/// The marker names the closure that was applied, and only an applied one.
|
||||
#[tokio::test]
|
||||
async fn swap_applies_a_closure_built_from_another_commit_with_a_warning() {
|
||||
async fn swap_marks_the_applied_closure_deployed_only_after_the_apply() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::down().built(OTHER_REV, true);
|
||||
let host = FakeHost::down();
|
||||
let id = node_id();
|
||||
let (result, log) = logging(run_swap(&host, &coord, "alice", id)).await;
|
||||
result.expect("drift is not a refusal");
|
||||
// The pins of the commit the closure came from are read before the apply
|
||||
// and recorded after it, in place of the ones this hive chose; the
|
||||
// bookkeeping node then marks those.
|
||||
run_swap(&host, &coord, "alice", id).await.expect("swap ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"recorded_rev alice".to_owned(),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("pins_at alice {OTHER_REV}"),
|
||||
format!("pins_at alice {REV}"),
|
||||
format!("pins_at alice {OTHER_REV}"),
|
||||
format!("swap_update alice {CLOSURE} node={}", id.get()),
|
||||
format!("record_meta alice {OTHER_REV}"),
|
||||
]
|
||||
host.calls().last().map(String::as_str),
|
||||
Some(format!("mark_deployed alice {CLOSURE}").as_str()),
|
||||
"{:?}",
|
||||
host.calls()
|
||||
);
|
||||
|
||||
let host = FakeHost::down().failing("swap_update");
|
||||
run_swap(&host, &coord, "alice", node_id())
|
||||
.await
|
||||
.expect_err("apply fails");
|
||||
assert!(
|
||||
!host.calls().iter().any(|c| c.starts_with("mark_deployed")),
|
||||
"{:?}",
|
||||
host.calls()
|
||||
);
|
||||
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
/// Pins that can't be read, or an apply that fails, leave the record alone.
|
||||
/// The container runs the closure, so a marker that cannot be written is a
|
||||
/// warning and not a failed swap.
|
||||
#[tokio::test]
|
||||
async fn a_drifted_swap_records_pins_only_for_an_apply_that_happened() {
|
||||
for failing in ["pins_at", "swap_update"] {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::down().built(OTHER_REV, true).failing(failing);
|
||||
run_swap(&host, &coord, "alice", node_id())
|
||||
.await
|
||||
.expect_err(failing);
|
||||
let calls = host.calls();
|
||||
assert!(
|
||||
!calls.iter().any(|c| c.starts_with("record_meta")),
|
||||
"{failing}: {calls:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.iter().any(|c| c.starts_with("swap_update")),
|
||||
failing == "swap_update",
|
||||
"{failing}: {calls:?}"
|
||||
);
|
||||
}
|
||||
async fn a_marker_that_cannot_be_written_does_not_fail_the_swap() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::down().failing("mark_deployed");
|
||||
let (result, log) = logging(run_swap(&host, &coord, "alice", node_id())).await;
|
||||
result.expect("swap ok");
|
||||
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
|
||||
}
|
||||
|
||||
/// A failed update fails the node and nothing else: the swap never restarts or
|
||||
|
|
@ -611,21 +489,8 @@ async fn a_failed_swap_reports_the_error_and_touches_no_container() {
|
|||
.expect_err("update failure propagates");
|
||||
assert!(format!("{err:#}").contains("update failed"), "{err:#}");
|
||||
let calls = host.calls();
|
||||
assert_eq!(calls.len(), 4, "{calls:?}");
|
||||
assert!(calls[3].starts_with("swap_update alice"), "{calls:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn swap_without_a_recorded_rev_never_updates() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost {
|
||||
rev: None,
|
||||
..FakeHost::down()
|
||||
};
|
||||
run_swap(&host, &coord, "alice", node_id())
|
||||
.await
|
||||
.expect_err("nothing to swap to");
|
||||
assert_eq!(host.calls(), ["recorded_rev alice"]);
|
||||
assert_eq!(calls.len(), 3, "{calls:?}");
|
||||
assert!(calls[2].starts_with("swap_update alice"), "{calls:?}");
|
||||
}
|
||||
|
||||
// --- Stop / StopForUpdate ---------------------------------------------------
|
||||
|
|
@ -851,7 +716,7 @@ async fn pause_drain_times_out_ok_and_clears_pending() {
|
|||
// --- MetaSync ---------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn meta_sync_prepares_then_resolves_and_records_last() {
|
||||
async fn meta_sync_only_prepares_the_rebuild() {
|
||||
let host = FakeHost::up();
|
||||
run_meta_sync(&host, "alice").await.expect("ok");
|
||||
assert_eq!(
|
||||
|
|
@ -860,8 +725,6 @@ async fn meta_sync_prepares_then_resolves_and_records_last() {
|
|||
"refresh_archetype alice".to_owned(),
|
||||
"prepare_rebuild_dirs alice".to_owned(),
|
||||
"refresh_agent_set".to_owned(),
|
||||
"resolve_meta alice".to_owned(),
|
||||
format!("record_meta alice {REV}"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
|
@ -870,17 +733,12 @@ async fn meta_sync_prepares_then_resolves_and_records_last() {
|
|||
async fn meta_sync_keeps_going_when_the_archetype_read_fails() {
|
||||
let host = FakeHost::up().failing("refresh_archetype");
|
||||
run_meta_sync(&host, "alice").await.expect("non-fatal");
|
||||
assert_eq!(host.calls().len(), 5);
|
||||
assert_eq!(host.calls().len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn meta_sync_stops_at_the_first_fatal_step() {
|
||||
for (fail, calls) in [
|
||||
("prepare_rebuild_dirs", 2),
|
||||
("refresh_agent_set", 3),
|
||||
("resolve_meta", 4),
|
||||
("record_meta", 5),
|
||||
] {
|
||||
for (fail, calls) in [("prepare_rebuild_dirs", 2), ("refresh_agent_set", 3)] {
|
||||
let host = FakeHost::up().failing(fail);
|
||||
run_meta_sync(&host, "alice")
|
||||
.await
|
||||
|
|
@ -913,11 +771,10 @@ async fn create_downloads_the_swarms_closure_and_creates_from_it() {
|
|||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"resolve_meta alice".to_owned(),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("create_only alice {CLOSURE}"),
|
||||
format!("mark_deployed alice {REV}"),
|
||||
format!("mark_deployed alice {CLOSURE}"),
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
|
|
@ -952,39 +809,11 @@ async fn a_create_without_a_closure_is_refused_and_logs_an_error() {
|
|||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_uses_a_closure_built_from_another_commit_with_a_warning() {
|
||||
let host = FakeHost::up().built(OTHER_REV, true);
|
||||
let (result, log) = logging(run_create(&host, "alice")).await;
|
||||
result.expect("drift is not a refusal");
|
||||
let calls = host.calls();
|
||||
// Created from the swarm's closure, and marked as the commit it was built
|
||||
// from, not the one this hive chose.
|
||||
assert_eq!(
|
||||
calls,
|
||||
[
|
||||
"resolve_meta alice".to_owned(),
|
||||
"closure_record alice".to_owned(),
|
||||
format!("realise alice {CLOSURE}"),
|
||||
format!("pins_at alice {OTHER_REV}"),
|
||||
format!("pins_at alice {REV}"),
|
||||
format!("pins_at alice {OTHER_REV}"),
|
||||
format!("create_only alice {CLOSURE}"),
|
||||
format!("mark_deployed alice {OTHER_REV}"),
|
||||
]
|
||||
);
|
||||
assert_eq!(log.at(tracing::Level::WARN).len(), 1);
|
||||
assert!(log.at(tracing::Level::ERROR).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_create_is_never_marked_deployed() {
|
||||
let host = FakeHost::up().failing("create_only");
|
||||
run_create(&host, "alice").await.expect_err("create fails");
|
||||
assert_eq!(host.calls().len(), 4);
|
||||
let host = FakeHost::up().failing("resolve_meta");
|
||||
run_create(&host, "alice").await.expect_err("resolve fails");
|
||||
assert_eq!(host.calls(), ["resolve_meta alice"]);
|
||||
assert_eq!(host.calls().len(), 3);
|
||||
}
|
||||
|
||||
// --- Start ------------------------------------------------------------------
|
||||
|
|
@ -1102,38 +931,9 @@ async fn destroy_bookkeeping_continues_past_a_failed_refresh() {
|
|||
// --- RebuildBookkeeping / EmitRebuilt ----------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebuild_bookkeeping_marks_the_recorded_build_deployed() {
|
||||
async fn rebuild_bookkeeping_rescans_without_touching_the_host() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::up();
|
||||
run_rebuild_bookkeeping(&host, &coord, "alice")
|
||||
.await
|
||||
.expect("ok");
|
||||
assert_eq!(
|
||||
host.calls(),
|
||||
[
|
||||
"recorded_build alice".to_owned(),
|
||||
format!("mark_deployed alice {REV}")
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebuild_bookkeeping_without_a_recording_marks_nothing() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::up().failing("recorded_build");
|
||||
run_rebuild_bookkeeping(&host, &coord, "alice")
|
||||
.await
|
||||
.expect("ok");
|
||||
assert_eq!(host.calls(), ["recorded_build alice"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_marker_write_does_not_fail_the_bookkeeping_node() {
|
||||
let (_dir, coord) = coordinator();
|
||||
let host = FakeHost::up().failing("mark_deployed");
|
||||
run_rebuild_bookkeeping(&host, &coord, "alice")
|
||||
.await
|
||||
.expect("warn only");
|
||||
run_rebuild_bookkeeping(&coord).await.expect("ok");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
|
@ -1234,30 +1034,21 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
|
|||
"refresh_archetype alice",
|
||||
"prepare_rebuild_dirs alice",
|
||||
"refresh_agent_set",
|
||||
"resolve_meta alice",
|
||||
"record_meta alice",
|
||||
],
|
||||
),
|
||||
(
|
||||
alice(|agent| NodeKind::Prebuild { agent }),
|
||||
&[
|
||||
"is_running alice",
|
||||
"recorded_rev alice",
|
||||
"closure_record alice",
|
||||
],
|
||||
&["is_running alice", "closure_record alice"],
|
||||
),
|
||||
(
|
||||
alice(|agent| NodeKind::Swap { agent }),
|
||||
&[
|
||||
"recorded_rev alice",
|
||||
"closure_record alice",
|
||||
"swap_update alice",
|
||||
"mark_deployed alice",
|
||||
],
|
||||
),
|
||||
(
|
||||
alice(|agent| NodeKind::RebuildBookkeeping { agent }),
|
||||
&["recorded_build alice", "mark_deployed alice"],
|
||||
),
|
||||
(alice(|agent| NodeKind::RebuildBookkeeping { agent }), &[]),
|
||||
(
|
||||
alice(|agent| NodeKind::Provision { agent }),
|
||||
&["provision alice"],
|
||||
|
|
@ -1265,7 +1056,6 @@ async fn dispatch_routes_each_host_backed_kind_to_its_executor() {
|
|||
(
|
||||
alice(|agent| NodeKind::Create { agent }),
|
||||
&[
|
||||
"resolve_meta alice",
|
||||
"closure_record alice",
|
||||
"create_only alice",
|
||||
"mark_deployed alice",
|
||||
|
|
|
|||
|
|
@ -190,6 +190,16 @@ impl JobQueue {
|
|||
Ok(named)
|
||||
}
|
||||
|
||||
/// Whether a rebuild of `agent` is queued or running: its `MetaSync` is not
|
||||
/// terminal yet.
|
||||
#[must_use]
|
||||
pub fn rebuild_live(&self, agent: &str) -> bool {
|
||||
self.lock().graph().nodes().any(|n| {
|
||||
!n.state.is_terminal()
|
||||
&& matches!(&n.payload, model::NodeKind::MetaSync { agent: a } if a == agent)
|
||||
})
|
||||
}
|
||||
|
||||
/// Queue `agent`'s first deploy unless one is already queued or running.
|
||||
///
|
||||
/// `None` means a `Provision` for the agent is still live. The check and
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ pub use hive_host_sock::jobs::State;
|
|||
use serde::Serialize;
|
||||
|
||||
/// The primitive operations — each kind maps to one executor fn in
|
||||
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs` / `meta_rev.rs`
|
||||
/// `exec.rs`, a thin wrapper over existing `lifecycle.rs`
|
||||
/// code. Concurrency is gated by resource classes (see
|
||||
/// [`Resource`](super::resource::Resource), declared per node where the node
|
||||
/// is constructed rather than derived from its kind).
|
||||
|
|
@ -29,8 +29,8 @@ use serde::Serialize;
|
|||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
#[strum(serialize_all = "snake_case")]
|
||||
pub enum NodeKind {
|
||||
/// The rebuild's preamble: runtime-dir prep and the `internal/meta`
|
||||
/// commit the build reads.
|
||||
/// The rebuild's preamble: runtime-dir prep, the archetype and agent-set
|
||||
/// refresh.
|
||||
MetaSync { agent: String },
|
||||
/// Download of the swarm's closure while the container keeps serving.
|
||||
Prebuild { agent: String },
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ fn stop_chain(
|
|||
wanted.guid()
|
||||
}
|
||||
|
||||
/// One agent's **start** subgraph. `SetWanted(Up)` head; a down + stale-rev
|
||||
/// agent gets the rebuild subgraph (its tail `Reconcile` starts it on
|
||||
/// One agent's **start** subgraph. `SetWanted(Up)` head; a down agent whose
|
||||
/// declared closure is not applied gets the rebuild subgraph (its tail `Reconcile` starts it on
|
||||
/// current derivations), otherwise a plain `Reconcile` (which starts a down
|
||||
/// agent and noops an already-running one).
|
||||
///
|
||||
|
|
@ -277,8 +277,8 @@ pub async fn restart_many(
|
|||
}
|
||||
|
||||
/// Start `agents` in a **single** DAG. A down agent gets
|
||||
/// `SetWanted(Up) → Reconcile` (or, rev stale, a rebuild-then-start so it comes
|
||||
/// up on current derivations); an already-running agent gets the same shape
|
||||
/// `SetWanted(Up) → Reconcile` (or, when the swarm declares a closure it has not applied, a
|
||||
/// rebuild-then-start so it comes up on the declared one); an already-running agent gets the same shape
|
||||
/// with the reconcile noop'ing.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
@ -287,14 +287,12 @@ pub async fn start_many(
|
|||
coord: &Arc<Coordinator>,
|
||||
agents: &[String],
|
||||
) -> anyhow::Result<Vec<NodeId>> {
|
||||
let current = crate::meta_rev::current_hyperhive_pin().await;
|
||||
let mut targets = Vec::with_capacity(agents.len());
|
||||
for agent in agents {
|
||||
let running = lifecycle::is_running(agent).await;
|
||||
let stored = read_rev_marker(&crate::paths::applied_rev_marker(agent));
|
||||
let stale = !crate::meta_rev::marker_is_fresh(current.as_deref(), stored.as_deref());
|
||||
let stale = crate::attic::closure::pending_path(agent).await.is_some();
|
||||
if !running && stale {
|
||||
tracing::info!(%agent, "start: rev stale + agent down — rebuild-then-start");
|
||||
tracing::info!(%agent, "start: closure declared + agent down — rebuild-then-start");
|
||||
}
|
||||
targets.push((agent.clone(), running, stale));
|
||||
}
|
||||
|
|
@ -303,20 +301,6 @@ pub async fn start_many(
|
|||
Ok(ids)
|
||||
}
|
||||
|
||||
/// The rev `marker` records, or `None` when there is none. A missing file is
|
||||
/// the normal never-applied case; any other read error is logged, and the
|
||||
/// caller still treats the rev as stale.
|
||||
fn read_rev_marker(marker: &std::path::Path) -> Option<String> {
|
||||
match std::fs::read_to_string(marker) {
|
||||
Ok(rev) => Some(rev),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %marker.display(), error = %e, "start: applied-rev marker unreadable; treating rev as stale");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop `agents` in a **single** DAG. A running agent gets
|
||||
/// `SetWanted(Off) → [Signal → Drain →](graceful) Reconcile`; a down agent
|
||||
/// skips the pointless quiesce but keeps the `Reconcile` as the race-up
|
||||
|
|
@ -366,19 +350,3 @@ pub fn pause_many(coord: &Arc<Coordinator>, agents: &[String]) -> anyhow::Result
|
|||
coord.emit_rebuild_queue_snapshot();
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::read_rev_marker;
|
||||
|
||||
#[test]
|
||||
fn a_rev_marker_reads_as_its_contents_and_none_when_absent_or_unreadable() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let marker = dir.path().join("rev");
|
||||
assert_eq!(read_rev_marker(&marker), None);
|
||||
std::fs::write(&marker, "abc").unwrap();
|
||||
assert_eq!(read_rev_marker(&marker).as_deref(), Some("abc"));
|
||||
// A directory where the file belongs fails with a non-NotFound error.
|
||||
assert_eq!(read_rev_marker(dir.path()), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1545,3 +1545,18 @@ fn a_first_deploy_is_not_queued_twice_for_one_agent() {
|
|||
"control: another agent's deploy is independent"
|
||||
);
|
||||
}
|
||||
|
||||
/// The auto-update pass queues no second rebuild for an agent that has one in
|
||||
/// flight; another agent's rebuild does not count.
|
||||
#[test]
|
||||
fn a_rebuild_is_live_for_its_own_agent_only() {
|
||||
let q = JobQueue::new(1);
|
||||
assert!(!q.rebuild_live("alice"), "nothing queued yet");
|
||||
q.insert_job(|b| templates::rebuild(b, "alice"))
|
||||
.expect("insert");
|
||||
assert!(
|
||||
q.rebuild_live("alice"),
|
||||
"control: the queued rebuild is seen"
|
||||
);
|
||||
assert!(!q.rebuild_live("bob"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -687,16 +687,6 @@ pub async fn list() -> Result<Vec<String>> {
|
|||
.collect())
|
||||
}
|
||||
|
||||
/// Logical agent names from a [`list`] result. An unreadable list stays an
|
||||
/// error rather than becoming "no agents": a caller fanning work out over
|
||||
/// every agent would otherwise report success after doing nothing.
|
||||
pub fn agent_names(listed: Result<Vec<String>>) -> Result<Vec<String>> {
|
||||
Ok(listed?
|
||||
.into_iter()
|
||||
.filter_map(|c| c.strip_prefix(AGENT_PREFIX).map(str::to_owned))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Ensure the per-agent runtime directory `/run/hyperhive/agents/<name>`
|
||||
/// exists. It is a bind source and `/run` is a tmpfs, so every start path
|
||||
/// creates it before `nixos-container start`.
|
||||
|
|
|
|||
|
|
@ -180,21 +180,6 @@ fn failed_destroy_with_an_unreadable_list_fails() {
|
|||
assert!(format!("{err:#}").contains("cannot confirm"), "{err:#}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_names_strips_the_container_prefix() {
|
||||
let names = agent_names(Ok(vec![format!("{AGENT_PREFIX}iris"), "other".to_owned()]))
|
||||
.expect("a readable list");
|
||||
assert_eq!(names, vec!["iris"]);
|
||||
}
|
||||
|
||||
/// A failed list is not an empty hive: the error reaches the caller.
|
||||
#[test]
|
||||
fn agent_names_propagates_an_unreadable_list() {
|
||||
let err = agent_names(Err(anyhow::anyhow!("connect to hive-priv socket")))
|
||||
.expect_err("an unreadable list must not read as zero agents");
|
||||
assert!(format!("{err:#}").contains("hive-priv"), "{err:#}");
|
||||
}
|
||||
|
||||
/// The setup path creates the agent's `work/` dir beside `state/`, so the
|
||||
/// bind source exists before nspawn starts the container.
|
||||
#[test]
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue