grant-bootstrap re-applies the granter and restarts failed policy units; setup treats a refused granting unit as not done
This commit is contained in:
parent
ae37c228b1
commit
c7d37ca455
8 changed files with 271 additions and 33 deletions
|
|
@ -116,7 +116,11 @@ It writes the `bao-bootstrap` policy, places a 24h grant token for
|
|||
`swarm-bao-granter-role` and restarts that unit. The token comes from the
|
||||
hidden prompt, or from stdin when stdin isn't a terminal; never from an
|
||||
argument or environment variable. If the granter exists already, it
|
||||
says so and does nothing. To do both steps in one go,
|
||||
runs the same steps again: this is also how a change to the granter's own
|
||||
policy or to `bao-bootstrap` reaches an existing store, because a deploy
|
||||
changes the text on the host and not in the store. Afterwards it resets and
|
||||
restarts every failed `swarm-bao-*-policy` unit, which otherwise stays down once
|
||||
it hits its start limit. To do both steps in one go,
|
||||
without re-typing the root token, run `swarmctl bao init --and-grant`
|
||||
instead (it needs the `pkcs11` seal, which unseals itself).
|
||||
|
||||
|
|
|
|||
|
|
@ -724,7 +724,7 @@ Prints the unseal or recovery keys and the root token to stdout, once, and store
|
|||
|
||||
Bootstrap the granter: the one-time step that lets `swarm-bao-granter-role` write every `swarm-*` policy and role.
|
||||
|
||||
Reads the root token from a hidden prompt, or from stdin when stdin isn't a terminal; never from an argument or an environment variable. Writes the `bao-bootstrap` policy, places a 24h grant token for the granter unit and restarts it, so run it as root on the host running that unit. Says so and does nothing when the granter exists already.
|
||||
Reads the root token from a hidden prompt, or from stdin when stdin isn't a terminal; never from an argument or an environment variable. Writes the `bao-bootstrap` policy, places a 24h grant token for the granter unit and restarts it, so run it as root on the host running that unit. When the granter exists already it runs the same steps again, which is how a change to the granter's policy reaches an existing store, then resets and restarts the failed `swarm-bao-*-policy` units.
|
||||
|
||||
**Usage:** `swarmctl bao grant-bootstrap [OPTIONS]`
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,13 @@ and the unit runs once. A successful run deletes the token file too.
|
|||
|
||||
After that, a new or changed `swarm-*` grant needs no operator step: the unit
|
||||
that writes it changes, and the deploy restarts it. A root step comes back only
|
||||
when the granter itself needs a path it lacks, such as a new mount.
|
||||
when the granter itself needs a path it lacks, such as a new mount. That
|
||||
step is `swarmctl bao grant-bootstrap`, which is also the re-apply: a change
|
||||
to `granterPolicyText` or to `bao-bootstrap-policy.hcl` reaches an existing
|
||||
store only through it, since the deploy leaves the store's copy alone. It
|
||||
writes `bao-bootstrap`, places a fresh 24h token, restarts the granter unit
|
||||
(which rewrites `bao-granter`), then resets and restarts the failed
|
||||
`swarm-bao-*-policy` units.
|
||||
|
||||
⚠️ The granting units re-run on **boot** and whenever a deploy **changes**
|
||||
them, not on every deploy. When a grant drifts in the store and its unit stays
|
||||
|
|
|
|||
|
|
@ -306,6 +306,8 @@ let
|
|||
# What a granting unit prints when the store refuses the granter: the
|
||||
# one-time step, runnable as root on this host.
|
||||
granterSetupSteps = [
|
||||
"swarmctl bao grant-bootstrap # does all of the following, and is also the re-apply after a policy change"
|
||||
"# on a host without swarmctl, run these instead:"
|
||||
"read -rs BAO_TOKEN && export BAO_TOKEN # the root token from 'bao operator init'"
|
||||
"bao policy write bao-bootstrap /etc/hyperhive/bao-bootstrap-policy.hcl"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -344,23 +344,47 @@ fn wait_unsealed(tools: &Tools) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// What `grant-bootstrap` did.
|
||||
#[derive(Debug)]
|
||||
enum Granted {
|
||||
AlreadyDone,
|
||||
Done,
|
||||
struct Granted {
|
||||
/// The granter already existed, so this re-applied the policies rather than
|
||||
/// bootstrapping for the first time.
|
||||
reapplied: bool,
|
||||
/// The failed `swarm-bao-*-policy` units that were reset and restarted
|
||||
/// once the granter unit had run.
|
||||
units: Vec<UnitRestart>,
|
||||
}
|
||||
|
||||
/// One failed policy unit and how its restart went.
|
||||
#[derive(Debug)]
|
||||
struct UnitRestart {
|
||||
name: String,
|
||||
/// Why it failed again, `None` when it is active now.
|
||||
failure: Option<String>,
|
||||
}
|
||||
|
||||
impl Granted {
|
||||
fn describe(&self, paths: &GrantPaths) -> String {
|
||||
match self {
|
||||
Self::AlreadyDone => format!(
|
||||
"the granter is already bootstrapped (policy {GRANTER_POLICY} exists); nothing done"
|
||||
),
|
||||
Self::Done => format!(
|
||||
let first = if self.reapplied {
|
||||
format!(
|
||||
"granter re-applied: {BOOTSTRAP_POLICY} written, grant token written to {}, \
|
||||
{GRANTER_UNIT} restarted to rewrite {GRANTER_POLICY} from the deployed policy",
|
||||
paths.token_file.display()
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"granter bootstrap started: grant token written to {}, {GRANTER_UNIT} restarted",
|
||||
paths.token_file.display()
|
||||
),
|
||||
}
|
||||
)
|
||||
};
|
||||
let units = self.units.iter().map(|unit| match &unit.failure {
|
||||
None => format!("{} was failed and is active again", unit.name),
|
||||
Some(why) => format!("{} is still failing: {why}", unit.name),
|
||||
});
|
||||
std::iter::once(first)
|
||||
.chain(units)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -370,12 +394,12 @@ impl Granted {
|
|||
fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result<Granted> {
|
||||
let policies = run(tools.bao_cmd(&["policy", "list"]).env("BAO_TOKEN", root))?;
|
||||
ensure(&policies, "bao policy list", &[root])?;
|
||||
if String::from_utf8_lossy(&policies.stdout)
|
||||
// A granter that exists is not a reason to stop: the deployed policy text
|
||||
// may be newer than the one in the store, and the granter unit is what
|
||||
// writes it. So the same steps run again.
|
||||
let reapplied = String::from_utf8_lossy(&policies.stdout)
|
||||
.lines()
|
||||
.any(|line| line.trim() == GRANTER_POLICY)
|
||||
{
|
||||
return Ok(Granted::AlreadyDone);
|
||||
}
|
||||
.any(|line| line.trim() == GRANTER_POLICY);
|
||||
|
||||
let write = run(tools
|
||||
.bao_cmd(&["policy", "write", BOOTSTRAP_POLICY])
|
||||
|
|
@ -422,7 +446,41 @@ fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result
|
|||
|
||||
let restart = run(Command::new(&tools.systemctl).args(["restart", GRANTER_UNIT]))?;
|
||||
ensure(&restart, "systemctl restart", &secrets)?;
|
||||
Ok(Granted::Done)
|
||||
let units = restart_failed_policy_units(tools, &secrets)?;
|
||||
Ok(Granted { reapplied, units })
|
||||
}
|
||||
|
||||
/// Resets and restarts every failed `swarm-bao-*-policy` unit. They retry on
|
||||
/// their own, but one that hit its start limit stays down, and a granter that
|
||||
/// was just re-applied is what lets them pass.
|
||||
fn restart_failed_policy_units(tools: &Tools, secrets: &[&str]) -> Result<Vec<UnitRestart>> {
|
||||
let listed = run(Command::new(&tools.systemctl).args([
|
||||
"list-units",
|
||||
"--state=failed",
|
||||
"--plain",
|
||||
"--no-legend",
|
||||
"--no-pager",
|
||||
"swarm-bao-*-policy.service",
|
||||
]))?;
|
||||
ensure(&listed, "systemctl list-units", secrets)?;
|
||||
let mut units = Vec::new();
|
||||
for name in String::from_utf8_lossy(&listed.stdout)
|
||||
.lines()
|
||||
.filter_map(|line| line.split_whitespace().next())
|
||||
.filter(|name| name.ends_with(".service"))
|
||||
{
|
||||
let reset = run(Command::new(&tools.systemctl).args(["reset-failed", name]))?;
|
||||
ensure(&reset, "systemctl reset-failed", secrets)?;
|
||||
let restarted = run(Command::new(&tools.systemctl).args(["restart", name]))?;
|
||||
let failure = ensure(&restarted, "systemctl restart", secrets)
|
||||
.err()
|
||||
.map(|e| format!("{e:#}"));
|
||||
units.push(UnitRestart {
|
||||
name: name.to_owned(),
|
||||
failure,
|
||||
});
|
||||
}
|
||||
Ok(units)
|
||||
}
|
||||
|
||||
impl Tools {
|
||||
|
|
@ -528,6 +586,9 @@ mod tests {
|
|||
paths: GrantPaths,
|
||||
}
|
||||
|
||||
/// The call that lists the failed policy units.
|
||||
const LIST_FAILED: &str = "systemctl list-units --state=failed --plain --no-legend --no-pager swarm-bao-*-policy.service";
|
||||
|
||||
const STUB: &str = r#"#!/bin/sh
|
||||
D=$(dirname "$0")
|
||||
echo "ARGV $(basename "$0") $*" >> "$D/log"
|
||||
|
|
@ -541,6 +602,12 @@ case "$(basename "$0") $1 $2 $3" in
|
|||
cat "$D/policies"; exit 0 ;;
|
||||
"bao policy write"*) exit 0 ;;
|
||||
"bao token create"*) echo "hvs.GRANTTOKEN"; exit 0 ;;
|
||||
"systemctl list-units --state=failed --plain")
|
||||
cat "$D/failed_units" 2>/dev/null; exit 0 ;;
|
||||
"systemctl reset-failed "*) exit 0 ;;
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service"*)
|
||||
if [ -e "$D/ssh_fails" ]; then echo "403 on sys/mounts/ssh-agents" >&2; exit 1; fi
|
||||
exit 0 ;;
|
||||
"systemctl restart "*)
|
||||
if [ -e "$D/restart_fails" ]; then echo "unit failed, token file: $(cat "$D/grant.token")" >&2; exit 5; fi
|
||||
exit 0 ;;
|
||||
|
|
@ -694,7 +761,7 @@ exit 9
|
|||
fn grant_runs_the_bootstrap_steps_in_order_with_no_token_in_any_argv() {
|
||||
let f = Fixture::new("grant-order");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert!(matches!(granted, Granted::Done));
|
||||
assert!(!granted.reapplied);
|
||||
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
|
|
@ -706,6 +773,7 @@ exit 9
|
|||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
let log = f.log();
|
||||
|
|
@ -785,15 +853,87 @@ exit 9
|
|||
);
|
||||
}
|
||||
|
||||
/// The documented re-run after a policy change: the granter exists, and the
|
||||
/// command still writes the policy, places the token and restarts the unit.
|
||||
#[test]
|
||||
fn grant_on_a_bootstrapped_store_changes_nothing() {
|
||||
fn grant_on_a_bootstrapped_store_applies_the_policies_again() {
|
||||
let f = Fixture::new("grant-again");
|
||||
f.set("policies", "bao-granter\ndefault\nroot\n");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("exits ok");
|
||||
assert!(matches!(granted, Granted::AlreadyDone));
|
||||
assert_eq!(f.calls(), ["bao policy list"]);
|
||||
assert!(!f.paths.token_file.exists());
|
||||
assert!(granted.describe(&f.paths).contains("already bootstrapped"));
|
||||
assert!(granted.reapplied);
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
f.calls(),
|
||||
[
|
||||
"bao policy list".to_owned(),
|
||||
format!("bao policy write bao-bootstrap {policy}"),
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan \
|
||||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(&f.paths.token_file).expect("the token is placed"),
|
||||
format!("{GRANT}\n")
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(said.contains("re-applied"), "{said}");
|
||||
assert!(!said.contains("nothing done"), "{said}");
|
||||
}
|
||||
|
||||
/// A unit that hit its start limit stays down on its own, so the command
|
||||
/// resets and restarts each failed one, after the granter unit has run.
|
||||
#[test]
|
||||
fn failed_policy_units_are_reset_and_restarted_after_the_granter() {
|
||||
let f = Fixture::new("grant-units");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert_eq!(
|
||||
f.calls()[3..],
|
||||
[
|
||||
"systemctl restart swarm-bao-granter-role",
|
||||
LIST_FAILED,
|
||||
"systemctl reset-failed swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl reset-failed swarm-bao-other-policy.service",
|
||||
"systemctl restart swarm-bao-other-policy.service",
|
||||
]
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A unit that fails again is reported, and does not undo the granter's
|
||||
/// success or stop the next unit.
|
||||
#[test]
|
||||
fn a_policy_unit_that_fails_again_is_reported_not_fatal() {
|
||||
let f = Fixture::new("grant-units-again");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
f.set("ssh_fails", "");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("the granter ran");
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service is still failing"),
|
||||
"{said}"
|
||||
);
|
||||
assert!(said.contains("403"), "{said}");
|
||||
assert!(
|
||||
said.contains("swarm-bao-other-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -840,7 +980,7 @@ exit 9
|
|||
);
|
||||
assert_eq!(calls[3], "bao policy list");
|
||||
assert_eq!(
|
||||
calls.last().map(String::as_str),
|
||||
calls.get(calls.len() - 2).map(String::as_str),
|
||||
Some("systemctl restart swarm-bao-granter-role")
|
||||
);
|
||||
assert!(calls.iter().all(|c| !c.contains(ROOT)), "{calls:?}");
|
||||
|
|
|
|||
|
|
@ -842,6 +842,7 @@ mod tests {
|
|||
"failed"
|
||||
}
|
||||
.to_owned(),
|
||||
denied: false,
|
||||
}]),
|
||||
admin_user: Ok(self.user),
|
||||
agents: Ok(Agents {
|
||||
|
|
|
|||
|
|
@ -333,8 +333,10 @@ enum BaoVerb {
|
|||
/// Reads the root token from a hidden prompt, or from stdin when stdin isn't
|
||||
/// a terminal; never from an argument or an environment variable. Writes
|
||||
/// the `bao-bootstrap` policy, places a 24h grant token for the granter unit
|
||||
/// and restarts it, so run it as root on the host running that unit. Says so
|
||||
/// and does nothing when the granter exists already.
|
||||
/// and restarts it, so run it as root on the host running that unit. When
|
||||
/// the granter exists already it runs the same steps again, which is how a
|
||||
/// change to the granter's policy reaches an existing store, then resets
|
||||
/// and restarts the failed `swarm-bao-*-policy` units.
|
||||
GrantBootstrap(bao::GrantBootstrapArgs),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,10 @@ pub(crate) struct Granter {
|
|||
pub(crate) struct Unit {
|
||||
pub(crate) name: String,
|
||||
pub(crate) state: String,
|
||||
/// The unit failed and its journal's last lines show the store refusing
|
||||
/// it with a 403 or "permission denied": the granter's policy in the store
|
||||
/// is older than the deployed one.
|
||||
pub(crate) denied: bool,
|
||||
}
|
||||
|
||||
/// One row of `GET /api/agents/status`, the two fields read.
|
||||
|
|
@ -165,7 +169,7 @@ const STEPS: [(&str, &str, bool); 10] = [
|
|||
pub(crate) fn classify(observed: &Observed) -> Vec<Step> {
|
||||
let unsealed = unsealed_status(&observed.store);
|
||||
let policy_units = policy_units_status(&observed.policy_units);
|
||||
let granter = granter_status(&observed.granter, &policy_units);
|
||||
let granter = granter_status(&observed.granter, &observed.policy_units, &policy_units);
|
||||
let reunsealed = match (&unsealed, &policy_units) {
|
||||
(Status::Done(_), Status::Done(_)) => Status::Done(String::new()),
|
||||
(Status::Unknown(e), _) | (_, Status::Unknown(e)) => Status::Unknown(e.clone()),
|
||||
|
|
@ -254,7 +258,23 @@ fn policy_units_status(units: &Result<Vec<Unit>, String>) -> Status {
|
|||
/// the token is absent, so after a reboot it is `inactive` whether or not it
|
||||
/// ever ran. An active granting unit logs in through the granter, so that
|
||||
/// also proves the granter exists.
|
||||
fn granter_status(granter: &Result<Granter, String>, policy_units: &Status) -> Status {
|
||||
fn granter_status(
|
||||
granter: &Result<Granter, String>,
|
||||
units: &Result<Vec<Unit>, String>,
|
||||
policy_units: &Status,
|
||||
) -> Status {
|
||||
// A granting unit the store refuses means the granter exists but is
|
||||
// behind: its upload in the journal and an active unit of its own say
|
||||
// nothing about the policy the store holds now.
|
||||
if let Ok(units) = units
|
||||
&& let Some(unit) = units.iter().find(|u| u.denied)
|
||||
{
|
||||
return Status::NotDone(format!(
|
||||
"{} was refused by the store (403): its granter policy is older than the deployed \
|
||||
one; `swarmctl bao grant-bootstrap` applies it",
|
||||
unit.name
|
||||
));
|
||||
}
|
||||
if matches!(policy_units, Status::Done(_)) {
|
||||
return Status::Done("the granting units log in through it".to_owned());
|
||||
}
|
||||
|
|
@ -655,13 +675,36 @@ fn policy_units() -> Result<Vec<Unit>, String> {
|
|||
Ok(names
|
||||
.into_iter()
|
||||
.zip(states)
|
||||
.map(|(name, state)| Unit {
|
||||
name,
|
||||
state: state.to_owned(),
|
||||
.map(|(name, state)| {
|
||||
let denied = state == "failed" && last_error_is_denied(&name);
|
||||
Unit {
|
||||
name,
|
||||
state: state.to_owned(),
|
||||
denied,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// How many of a failed unit's last journal lines are read for its error.
|
||||
const JOURNAL_TAIL: &str = "30";
|
||||
|
||||
/// Whether the unit's last journal lines show the store refusing it. An
|
||||
/// unreadable journal counts as no: the unit's state decides.
|
||||
fn last_error_is_denied(unit: &str) -> bool {
|
||||
run(Command::new("journalctl").args(["-u", unit, "-n", JOURNAL_TAIL, "--no-pager", "-q"]))
|
||||
.is_ok_and(|j| journal_shows_denial(&j.stdout))
|
||||
}
|
||||
|
||||
/// A 403 or "permission denied" in `journal`, as the `bao` CLI prints a refusal
|
||||
/// (`Code: 403` and `permission denied`).
|
||||
fn journal_shows_denial(journal: &str) -> bool {
|
||||
journal.lines().any(|line| {
|
||||
let line = line.to_ascii_lowercase();
|
||||
line.contains("permission denied") || line.contains("code: 403")
|
||||
})
|
||||
}
|
||||
|
||||
/// The unit names in `systemctl list-unit-files --no-legend`, without the one
|
||||
/// that waits on authelia.
|
||||
fn parse_unit_files(listed: &str) -> Vec<String> {
|
||||
|
|
@ -742,6 +785,7 @@ mod tests {
|
|||
.map(|(i, s)| Unit {
|
||||
name: format!("swarm-bao-u{i}-policy.service"),
|
||||
state: (*s).to_owned(),
|
||||
denied: false,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -882,6 +926,45 @@ mod tests {
|
|||
assert_eq!(kind(&status_of(&o, 3)), "done");
|
||||
}
|
||||
|
||||
/// A granting unit the store refuses with a 403 means the granter exists but
|
||||
/// its policy in the store is behind the deployed one: an upload in its
|
||||
/// journal and an active granter unit don't make step 3 done.
|
||||
#[test]
|
||||
fn a_granting_unit_refused_with_a_403_means_the_granter_is_not_done() {
|
||||
let mut o = finished();
|
||||
o.granter = Ok(Granter {
|
||||
state: "active".to_owned(),
|
||||
uploaded: true,
|
||||
});
|
||||
o.policy_units = Ok(units(&["active", "failed"]));
|
||||
// The control: a failed unit that is not a refusal leaves the granter done.
|
||||
assert_eq!(kind(&status_of(&o, 3)), "done");
|
||||
if let Ok(units) = &mut o.policy_units {
|
||||
units[1].denied = true;
|
||||
}
|
||||
assert!(matches!(
|
||||
status_of(&o, 3),
|
||||
Status::NotDone(why) if why.contains("u1-policy") && why.contains("swarmctl bao grant-bootstrap")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_is_read_from_the_journal_as_the_bao_cli_prints_it() {
|
||||
let refused = "Error enabling: Error making API request.\n\n\
|
||||
URL: POST https://127.0.0.1:8200/v1/sys/mounts/ssh-agents\n\
|
||||
Code: 403. Errors:\n\n* 1 error occurred:\n\t* permission denied\n";
|
||||
assert!(journal_shows_denial(refused));
|
||||
assert!(journal_shows_denial("something\nCode: 403. Errors:"));
|
||||
// The controls: other failures, and nothing at all.
|
||||
assert!(!journal_shows_denial(
|
||||
"dial tcp 127.0.0.1:8200: connection refused\n"
|
||||
));
|
||||
assert!(!journal_shows_denial(
|
||||
"the store is sealed or unreachable; retrying.\n"
|
||||
));
|
||||
assert!(!journal_shows_denial(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_inactive_policy_unit_fails_step_four_and_names_it() {
|
||||
let mut o = finished();
|
||||
|
|
|
|||
Loading…
Reference in a new issue