537 lines
20 KiB
Rust
537 lines
20 KiB
Rust
//! Unit tests for the lifecycle module (moved verbatim from the old
|
|
//! single-file `lifecycle.rs` `#[cfg(test)]` block).
|
|
|
|
use super::*;
|
|
|
|
/// Regression test: `setup_applied` must seed both agent.nix and flake.nix
|
|
/// in the initial commit, and tag it `deployed/0`. Before commit 5b5a93e
|
|
/// flake.nix was missing from the scaffold, requiring manual creation (seen
|
|
/// with the damocles agent).
|
|
///
|
|
/// Exercises the template arm: there is no forge to take a config from in a
|
|
/// test (no reader token), so `setup_applied` falls through to
|
|
/// `seed_template` — which is the arm this asserts about.
|
|
#[tokio::test]
|
|
async fn setup_applied_seeds_flake_nix() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let applied = dir.path().join("applied");
|
|
setup_applied(&applied, "test-agent")
|
|
.await
|
|
.expect("setup_applied");
|
|
git_rev_parse(&applied, "refs/tags/deployed/0")
|
|
.await
|
|
.expect("the seed commit is tagged deployed/0");
|
|
|
|
// Both files must exist on disk.
|
|
assert!(applied.join("agent.nix").exists(), "agent.nix missing");
|
|
assert!(applied.join("flake.nix").exists(), "flake.nix missing");
|
|
|
|
// flake.nix must export nixosModules.default (the meta-flake contract).
|
|
let flake = std::fs::read_to_string(applied.join("flake.nix")).unwrap();
|
|
assert!(
|
|
flake.contains("nixosModules.default"),
|
|
"flake.nix does not export nixosModules.default"
|
|
);
|
|
|
|
// Both files must be tracked in the initial git commit.
|
|
let out = git_command()
|
|
.current_dir(&applied)
|
|
.args(["show", "--name-only", "--format=", "HEAD"])
|
|
.output()
|
|
.await
|
|
.expect("git show");
|
|
let tracked = String::from_utf8_lossy(&out.stdout);
|
|
assert!(tracked.contains("agent.nix"), "agent.nix not committed");
|
|
assert!(tracked.contains("flake.nix"), "flake.nix not committed");
|
|
}
|
|
|
|
#[test]
|
|
fn bridge_gateway_ip_extracts_verbatim_address() {
|
|
// HIVE_NETWORK_SUBNET carries the bridge IP verbatim, not the
|
|
// canonical network — the gateway is the address before the `/`.
|
|
assert_eq!(
|
|
bridge_gateway_ip("10.42.0.1/24").as_deref(),
|
|
Some("10.42.0.1")
|
|
);
|
|
// Non-`.1` operator override: the gateway is wherever the bridge is.
|
|
assert_eq!(
|
|
bridge_gateway_ip("10.42.0.254/24").as_deref(),
|
|
Some("10.42.0.254")
|
|
);
|
|
assert_eq!(
|
|
bridge_gateway_ip("172.30.0.1/16").as_deref(),
|
|
Some("172.30.0.1")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn bridge_gateway_ip_rejects_bad_input() {
|
|
assert!(bridge_gateway_ip("notanip/24").is_none());
|
|
assert!(bridge_gateway_ip("10.42.0.1").is_none()); // no prefix
|
|
assert!(bridge_gateway_ip("10.42.0.1/33").is_none()); // prefix > 32
|
|
assert!(bridge_gateway_ip("10.42.0.999/24").is_none()); // octet > 255
|
|
assert!(bridge_gateway_ip("10.42.0/24").is_none()); // 3 octets
|
|
}
|
|
|
|
/// The presence control for the two rejection tests below: with both
|
|
/// variables set and well-formed, the settings are built. Without this,
|
|
/// a `network_isolation_from_vars` that rejected *everything* would pass
|
|
/// every absence assertion and look like a working guard.
|
|
#[test]
|
|
fn network_isolation_accepts_a_well_formed_pair() {
|
|
let iso = network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/24"))
|
|
.expect("well-formed bridge + subnet must be accepted");
|
|
assert_eq!(iso.bridge, "hive0");
|
|
// The gateway is the verbatim bridge address, prefix stripped.
|
|
assert_eq!(iso.gateway_ip, "10.42.0.1");
|
|
}
|
|
|
|
/// A missing or empty variable is fatal, not a fallback to the host
|
|
/// netns. Empty is tested alongside unset because `std::env::var` on a
|
|
/// variable set to `""` returns `Ok("")`, so treating only `None` as
|
|
/// missing would let an empty value through.
|
|
#[test]
|
|
fn network_isolation_rejects_missing_or_empty_vars() {
|
|
assert!(network_isolation_from_vars(None, Some("10.42.0.1/24")).is_err());
|
|
assert!(network_isolation_from_vars(Some("hive0"), None).is_err());
|
|
assert!(network_isolation_from_vars(None, None).is_err());
|
|
assert!(network_isolation_from_vars(Some(""), Some("10.42.0.1/24")).is_err());
|
|
assert!(network_isolation_from_vars(Some("hive0"), Some("")).is_err());
|
|
}
|
|
|
|
/// A malformed subnet is fatal too. Previously this logged a warning and
|
|
/// silently produced a container on the host netns — a dropped security
|
|
/// boundary with nothing in the journal saying so.
|
|
#[test]
|
|
fn network_isolation_rejects_a_malformed_subnet() {
|
|
assert!(network_isolation_from_vars(Some("hive0"), Some("notanip/24")).is_err());
|
|
assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1")).is_err());
|
|
assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.1/33")).is_err());
|
|
assert!(network_isolation_from_vars(Some("hive0"), Some("10.42.0.999/24")).is_err());
|
|
}
|
|
|
|
/// The error has to name the variable an operator must fix — these are
|
|
/// read at startup, so the message is the whole diagnostic.
|
|
#[test]
|
|
fn network_isolation_errors_name_the_offending_variable() {
|
|
let e = network_isolation_from_vars(None, Some("10.42.0.1/24")).unwrap_err();
|
|
assert!(
|
|
format!("{e:#}").contains("HIVE_NETWORK_BRIDGE"),
|
|
"bridge error must name the variable, got: {e:#}"
|
|
);
|
|
let e = network_isolation_from_vars(Some("hive0"), None).unwrap_err();
|
|
assert!(
|
|
format!("{e:#}").contains("HIVE_NETWORK_SUBNET"),
|
|
"subnet error must name the variable, got: {e:#}"
|
|
);
|
|
let e = network_isolation_from_vars(Some("hive0"), Some("nope/24")).unwrap_err();
|
|
let msg = format!("{e:#}");
|
|
assert!(
|
|
msg.contains("HIVE_NETWORK_SUBNET") && msg.contains("nope/24"),
|
|
"malformed-subnet error must name the variable and the bad value, got: {msg}"
|
|
);
|
|
}
|
|
|
|
/// `setup_applied` is idempotent: calling it on an already seeded repo is a
|
|
/// no-op (the shape check passes and nothing is written).
|
|
#[tokio::test]
|
|
async fn setup_applied_idempotent() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let applied = dir.path().join("applied");
|
|
setup_applied(&applied, "test-agent")
|
|
.await
|
|
.expect("first call");
|
|
// Second call must not error even though .git already exists.
|
|
setup_applied(&applied, "test-agent")
|
|
.await
|
|
.expect("second call");
|
|
// Still one commit.
|
|
let out = git_command()
|
|
.current_dir(&applied)
|
|
.args(["rev-list", "--count", "HEAD"])
|
|
.output()
|
|
.await
|
|
.expect("git rev-list");
|
|
let count = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
|
assert_eq!(
|
|
count, "1",
|
|
"expected exactly one commit after idempotent call"
|
|
);
|
|
}
|
|
|
|
/// Build a two-commit repo and return `(dir, repo_path, first, second)`.
|
|
async fn two_commit_repo() -> (tempfile::TempDir, std::path::PathBuf, String, String) {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let repo = dir.path().join("applied");
|
|
setup_applied(&repo, "test-agent")
|
|
.await
|
|
.expect("setup_applied");
|
|
let first = git_rev_parse(&repo, "HEAD").await.expect("rev-parse first");
|
|
|
|
std::fs::write(repo.join("second.txt"), "second").expect("write second.txt");
|
|
git(&repo, &["add", "second.txt"]).await.expect("git add");
|
|
git(
|
|
&repo,
|
|
&[
|
|
"-c",
|
|
"user.name=test",
|
|
"-c",
|
|
"user.email=test@example.com",
|
|
"commit",
|
|
"-m",
|
|
"second",
|
|
],
|
|
)
|
|
.await
|
|
.expect("git commit");
|
|
let second = git_rev_parse(&repo, "HEAD")
|
|
.await
|
|
.expect("rev-parse second");
|
|
|
|
assert_ne!(first, second, "second commit did not advance HEAD");
|
|
(dir, repo, first, second)
|
|
}
|
|
|
|
/// `git_is_ancestor` answers reachability in both directions. This is the
|
|
/// check that decides whether moving a branch is a real fast-forward or a
|
|
/// rewind that discards commits.
|
|
#[tokio::test]
|
|
async fn is_ancestor_distinguishes_direction() {
|
|
let (_dir, repo, first, second) = two_commit_repo().await;
|
|
|
|
assert!(
|
|
git_is_ancestor(&repo, &first, &second)
|
|
.await
|
|
.expect("ancestor check forward"),
|
|
"first must be an ancestor of second"
|
|
);
|
|
assert!(
|
|
!git_is_ancestor(&repo, &second, &first)
|
|
.await
|
|
.expect("ancestor check reverse"),
|
|
"second must NOT be an ancestor of first"
|
|
);
|
|
}
|
|
|
|
/// Regression test for the deploy path that ate a committed agent config:
|
|
/// `git_update_ref_cas` must refuse to move a ref whose current value is not
|
|
/// the expected one, and must leave the ref untouched when it refuses. A bare
|
|
/// `update-ref` accepts that move and silently drops everything in between.
|
|
#[tokio::test]
|
|
async fn update_ref_cas_refuses_stale_expectation() {
|
|
let (_dir, repo, first, second) = two_commit_repo().await;
|
|
|
|
// Park a ref at `first`, then CAS it forward with the correct old value.
|
|
git_update_ref(&repo, "refs/heads/cas", &first)
|
|
.await
|
|
.expect("plant ref");
|
|
git_update_ref_cas(&repo, "refs/heads/cas", &second, &first)
|
|
.await
|
|
.expect("CAS with the correct old value must succeed");
|
|
assert_eq!(
|
|
git_rev_parse(&repo, "refs/heads/cas").await.unwrap(),
|
|
second
|
|
);
|
|
|
|
// Now retry with the SAME (now stale) expectation, as a racing deploy
|
|
// holding a pre-merge sha would: it must fail rather than rewind.
|
|
assert!(
|
|
git_update_ref_cas(&repo, "refs/heads/cas", &first, &first)
|
|
.await
|
|
.is_err(),
|
|
"CAS accepted a stale old value"
|
|
);
|
|
assert_eq!(
|
|
git_rev_parse(&repo, "refs/heads/cas").await.unwrap(),
|
|
second,
|
|
"ref moved even though the CAS failed"
|
|
);
|
|
}
|
|
|
|
/// `systemctl is-active` prints the state with a trailing newline, which is
|
|
/// the whole reason this parser trims rather than comparing raw.
|
|
#[test]
|
|
fn unit_state_reads_what_is_active_prints() {
|
|
assert_eq!(UnitState::from_is_active("active\n"), UnitState::Active);
|
|
assert_eq!(UnitState::from_is_active("failed\n"), UnitState::Failed);
|
|
assert_eq!(UnitState::from_is_active("inactive\n"), UnitState::Other);
|
|
// No trailing newline, in case the capture ever changes shape.
|
|
assert_eq!(UnitState::from_is_active("failed"), UnitState::Failed);
|
|
}
|
|
|
|
/// The asymmetry that matters: an unrecognised state must fall to `Other`,
|
|
/// never to `Failed`.
|
|
///
|
|
/// `Failed` is what the dashboard will render as *this agent gave up*, so a
|
|
/// wrong `Failed` invents an incident, while a wrong `Other` merely fails to
|
|
/// distinguish one from a deliberate stop — the behaviour we have today.
|
|
/// Guessing in the safe direction is the property, not an implementation
|
|
/// detail of the `match`.
|
|
#[test]
|
|
fn an_unknown_state_is_never_reported_as_failed() {
|
|
for s in [
|
|
"activating",
|
|
"deactivating",
|
|
"reloading",
|
|
"maintenance",
|
|
"unknown",
|
|
"",
|
|
"Failed", // capitalised: systemd prints lowercase, so this is not a state
|
|
"failed*", // `is-active` never emits this; a substring match would take it
|
|
] {
|
|
assert_eq!(
|
|
UnitState::from_is_active(s),
|
|
UnitState::Other,
|
|
"{s:?} must not be read as a give-up"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// A failed `nixos-container destroy` whose container is still listed fails,
|
|
/// and keeps hive-priv's error in the chain. The destroy DAG gates the state
|
|
/// purge on this call, so this is the case that must not read as success.
|
|
#[test]
|
|
fn failed_destroy_with_the_container_still_listed_fails() {
|
|
let err = confirm_gone_after_failed_destroy(
|
|
anyhow::anyhow!("nixos-container destroy h-doomed failed (exit status: 1)"),
|
|
"h-doomed",
|
|
Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
|
|
)
|
|
.expect_err("a surviving container must fail the destroy");
|
|
let chain = format!("{err:#}");
|
|
assert!(chain.contains("h-doomed still exists"), "{chain}");
|
|
assert!(chain.contains("exit status: 1"), "{chain}");
|
|
}
|
|
|
|
/// A container missing from the list is gone, whatever the destroy exit code
|
|
/// said, so the destroy succeeds.
|
|
#[test]
|
|
fn failed_destroy_with_the_container_gone_succeeds() {
|
|
confirm_gone_after_failed_destroy(
|
|
anyhow::anyhow!("nixos-container destroy h-doomed failed"),
|
|
"h-doomed",
|
|
Ok(vec!["h-other".to_owned()]),
|
|
)
|
|
.expect("an absent container is a completed destroy");
|
|
// Control: the same list with the container in it fails, so the pass
|
|
// above comes from the absence, not from the helper accepting anything.
|
|
assert!(
|
|
confirm_gone_after_failed_destroy(
|
|
anyhow::anyhow!("nixos-container destroy h-doomed failed"),
|
|
"h-doomed",
|
|
Ok(vec!["h-other".to_owned(), "h-doomed".to_owned()]),
|
|
)
|
|
.is_err()
|
|
);
|
|
}
|
|
|
|
/// An unreadable list proves nothing about the container, so the destroy
|
|
/// fails rather than guessing it is gone.
|
|
#[test]
|
|
fn failed_destroy_with_an_unreadable_list_fails() {
|
|
let err = confirm_gone_after_failed_destroy(
|
|
anyhow::anyhow!("nixos-container destroy h-doomed failed"),
|
|
"h-doomed",
|
|
Err(anyhow::anyhow!("connect to hive-priv socket")),
|
|
)
|
|
.expect_err("an unreadable list must not pass for an absent container");
|
|
assert!(format!("{err:#}").contains("cannot confirm"), "{err:#}");
|
|
}
|
|
|
|
#[test]
|
|
fn agent_names_strips_the_container_prefix() {
|
|
let names = agent_names(Ok(vec![format!("{AGENT_PREFIX}iris"), "other".to_owned()]))
|
|
.expect("a readable list");
|
|
assert_eq!(names, vec!["iris"]);
|
|
}
|
|
|
|
/// A failed list is not an empty hive: the error reaches the caller.
|
|
#[test]
|
|
fn agent_names_propagates_an_unreadable_list() {
|
|
let err = agent_names(Err(anyhow::anyhow!("connect to hive-priv socket")))
|
|
.expect_err("an unreadable list must not read as zero agents");
|
|
assert!(format!("{err:#}").contains("hive-priv"), "{err:#}");
|
|
}
|
|
|
|
/// The setup path creates the agent's `work/` dir beside `state/`, so the
|
|
/// bind source exists before nspawn starts the container.
|
|
#[test]
|
|
fn ensure_state_dir_creates_the_work_dir() {
|
|
let root = tempfile::tempdir().expect("tempdir");
|
|
let notes_dir = root.path().join("state");
|
|
ensure_state_dir(¬es_dir).expect("ensure_state_dir");
|
|
assert!(notes_dir.is_dir());
|
|
assert!(root.path().join(crate::paths::WORK_SUBDIR).is_dir());
|
|
}
|
|
|
|
/// Real build failures (`error:`) and both of nix's own warning spellings
|
|
/// must stay WARN; routine progress chatter is the only thing demoted.
|
|
/// Regression guard for a first-pass predicate that only matched
|
|
/// `warning:` and silently buried `error:` lines at INFO.
|
|
#[test]
|
|
fn is_actionable_nix_line_keeps_warnings_and_errors_demotes_chatter() {
|
|
let actionable = [
|
|
"warning: Lock file entry is unlocked",
|
|
" warning: leading whitespace",
|
|
"evaluation warning: this will be an error in a future release",
|
|
"error: builder for '/nix/store/foo.drv' failed with exit code 1",
|
|
"error: 1 dependencies of derivation '/nix/store/bar.drv' failed to build",
|
|
];
|
|
for line in actionable {
|
|
assert!(is_actionable_nix_line(line), "{line}");
|
|
}
|
|
|
|
let chatter = [
|
|
"copying 3 paths",
|
|
"these 2 derivations will be built:",
|
|
"fetching git input 'git+file:///var/lib/hyperhive'",
|
|
"structuredAttrs is enabled",
|
|
];
|
|
for line in chatter {
|
|
assert!(!is_actionable_nix_line(line), "{line}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_hung_systemctl_reads_as_unknown_within_the_bound() {
|
|
let started = tokio::time::Instant::now();
|
|
let got = output_within(
|
|
std::time::Duration::from_millis(20),
|
|
std::future::pending::<std::io::Result<()>>(),
|
|
&["is-active"],
|
|
)
|
|
.await;
|
|
assert!(got.is_none());
|
|
assert!(started.elapsed() < std::time::Duration::from_secs(5));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a_systemctl_answer_inside_the_bound_passes_through() {
|
|
let got = output_within(
|
|
std::time::Duration::from_secs(5),
|
|
async { Ok(7_u8) },
|
|
&["is-active"],
|
|
)
|
|
.await;
|
|
assert_eq!(got, Some(7));
|
|
}
|
|
|
|
/// A child that outlives the deadline yields an error naming the command and
|
|
/// the deadline, and the child and its grandchild are both killed rather than
|
|
/// orphaned.
|
|
#[tokio::test]
|
|
async fn run_with_deadline_times_out_and_kills_the_process_group() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let (shpid, sleeppid) = (dir.path().join("sh.pid"), dir.path().join("sleep.pid"));
|
|
let mut cmd = tokio::process::Command::new("sh");
|
|
cmd.arg("-c").arg(format!(
|
|
"echo $$ > {}; sleep 300 & echo $! > {}; wait",
|
|
shpid.display(),
|
|
sleeppid.display()
|
|
));
|
|
let err = run_with_deadline(
|
|
&mut cmd,
|
|
std::time::Duration::from_millis(500),
|
|
"fake fetch",
|
|
)
|
|
.await
|
|
.expect_err("a sleeping child must hit the deadline");
|
|
let msg = format!("{err:#}");
|
|
assert!(msg.contains("fake fetch timed out after 0.5s"), "{msg}");
|
|
|
|
for file in [&shpid, &sleeppid] {
|
|
// The pid file proves the process existed before the kill.
|
|
let pid = std::fs::read_to_string(file)
|
|
.expect("process wrote its pid before the deadline")
|
|
.trim()
|
|
.to_owned();
|
|
let proc = std::path::PathBuf::from("/proc").join(&pid);
|
|
let mut gone = false;
|
|
for _ in 0..100 {
|
|
if !proc.exists() {
|
|
gone = true;
|
|
break;
|
|
}
|
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
|
}
|
|
assert!(gone, "pid {pid} ({}) still running", file.display());
|
|
}
|
|
}
|
|
|
|
/// Presence control for the timeout test: a child that finishes in time
|
|
/// returns its output.
|
|
#[tokio::test]
|
|
async fn run_with_deadline_returns_output_of_a_fast_child() {
|
|
let mut cmd = tokio::process::Command::new("echo");
|
|
cmd.arg("hi");
|
|
let out = run_with_deadline(&mut cmd, std::time::Duration::from_secs(30), "echo")
|
|
.await
|
|
.expect("echo finishes");
|
|
assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "hi");
|
|
}
|
|
|
|
/// The per-host values reach the container as facts: the dashboard port and
|
|
/// pinned claude from the hive's config, the OTEL labels and the queue
|
|
/// address from the daemon's environment.
|
|
#[test]
|
|
fn hive_facts_carry_the_per_host_values() {
|
|
let hive = HiveEnv {
|
|
dashboard_port: 7123,
|
|
claude_code_path: Some("/nix/store/0abc-claude-code".to_owned()),
|
|
..HiveEnv::default()
|
|
};
|
|
let facts = hive_facts_from(&hive, |name| match name {
|
|
"HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES" => Some("team=a".to_owned()),
|
|
"HIVE_AGENT_NATS_URL" => Some("tls://nats.t.local:4222".to_owned()),
|
|
"HIVE_SQUID_PROXY" => Some("10.42.0.1:3128".to_owned()),
|
|
"HIVE_EGRESS_NO_PROXY" => Some("10.42.0.1,t.local".to_owned()),
|
|
_ => None,
|
|
});
|
|
assert_eq!(facts.dashboard_port, Some(7123));
|
|
assert_eq!(
|
|
facts.claude_code_path.as_deref(),
|
|
Some("/nix/store/0abc-claude-code")
|
|
);
|
|
assert_eq!(
|
|
facts.otel_extra_resource_attributes.as_deref(),
|
|
Some("team=a")
|
|
);
|
|
assert_eq!(
|
|
facts.agent_nats_url.as_deref(),
|
|
Some("tls://nats.t.local:4222")
|
|
);
|
|
assert_eq!(facts.squid_proxy.as_deref(), Some("10.42.0.1:3128"));
|
|
assert_eq!(facts.egress_no_proxy.as_deref(), Some("10.42.0.1,t.local"));
|
|
}
|
|
|
|
/// Unset and empty both leave a fact out, so the agent falls back to its
|
|
/// own default rather than reading an empty value.
|
|
#[test]
|
|
fn hive_facts_leave_unset_values_out() {
|
|
let facts = hive_facts_from(&HiveEnv::default(), |name| {
|
|
(name == "HYPERHIVE_OTEL_EXTRA_RESOURCE_ATTRIBUTES").then(String::new)
|
|
});
|
|
assert_eq!(facts.claude_code_path, None);
|
|
assert_eq!(facts.otel_extra_resource_attributes, None);
|
|
assert_eq!(facts.agent_nats_url, None);
|
|
assert_eq!(facts.squid_proxy, None);
|
|
assert_eq!(facts.egress_no_proxy, None);
|
|
assert_eq!(facts.name, None);
|
|
}
|
|
|
|
/// A failed `start` leaves its error in the op's build-log row; without it the
|
|
/// row is empty. A missing row id records nothing and does not panic.
|
|
#[test]
|
|
fn a_failed_start_puts_its_error_in_the_op_output() {
|
|
let dir = tempfile::tempdir().expect("tempdir");
|
|
let logs = crate::build_logs::BuildLogs::open(dir.path()).expect("open");
|
|
let id = logs
|
|
.start("updo", "start", "nixos-container start h-updo", None)
|
|
.expect("start");
|
|
let err = anyhow::anyhow!("nixos-container start h-updo failed (exit status: 1): Job failed");
|
|
record_failure(Some(&logs), Some(id), &err);
|
|
let full = logs.get_full(id).expect("get").expect("row");
|
|
assert!(full.stderr.contains("Job failed"), "{}", full.stderr);
|
|
record_failure(Some(&logs), None, &err);
|
|
record_failure(None, Some(id), &err);
|
|
}
|