Watch
0
0
Fork
You've already forked hyperhive
0

grant-bootstrap re-applies the granter and restarts failed policy units; setup treats a refused granting unit as not done

This commit is contained in:
maia 2026-11-19 06:18:04 +01:00 • committed by mara
commit c7d37ca455

View file

@ -344,23 +344,47 @@ fn wait_unsealed(tools: &Tools) -> Result<()> {
}
}
/// What `grant-bootstrap` did.
#[derive(Debug)]
enum Granted {
AlreadyDone,
Done,
struct Granted {
/// The granter already existed, so this re-applied the policies rather than
/// bootstrapping for the first time.
reapplied: bool,
/// The failed `swarm-bao-*-policy` units that were reset and restarted
/// once the granter unit had run.
units: Vec<UnitRestart>,
}
/// One failed policy unit and how its restart went.
#[derive(Debug)]
struct UnitRestart {
name: String,
/// Why it failed again, `None` when it is active now.
failure: Option<String>,
}
impl Granted {
fn describe(&self, paths: &GrantPaths) -> String {
match self {
Self::AlreadyDone => format!(
"the granter is already bootstrapped (policy {GRANTER_POLICY} exists); nothing done"
),
Self::Done => format!(
let first = if self.reapplied {
format!(
"granter re-applied: {BOOTSTRAP_POLICY} written, grant token written to {}, \
{GRANTER_UNIT} restarted to rewrite {GRANTER_POLICY} from the deployed policy",
paths.token_file.display()
)
} else {
format!(
"granter bootstrap started: grant token written to {}, {GRANTER_UNIT} restarted",
paths.token_file.display()
),
}
)
};
let units = self.units.iter().map(|unit| match &unit.failure {
None => format!("{} was failed and is active again", unit.name),
Some(why) => format!("{} is still failing: {why}", unit.name),
});
std::iter::once(first)
.chain(units)
.collect::<Vec<_>>()
.join("\n")
}
}
@ -370,12 +394,12 @@ impl Granted {
fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result<Granted> {
let policies = run(tools.bao_cmd(&["policy", "list"]).env("BAO_TOKEN", root))?;
ensure(&policies, "bao policy list", &[root])?;
if String::from_utf8_lossy(&policies.stdout)
// A granter that exists is not a reason to stop: the deployed policy text
// may be newer than the one in the store, and the granter unit is what
// writes it. So the same steps run again.
let reapplied = String::from_utf8_lossy(&policies.stdout)
.lines()
.any(|line| line.trim() == GRANTER_POLICY)
{
return Ok(Granted::AlreadyDone);
}
.any(|line| line.trim() == GRANTER_POLICY);
let write = run(tools
.bao_cmd(&["policy", "write", BOOTSTRAP_POLICY])
@ -422,7 +446,41 @@ fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result
let restart = run(Command::new(&tools.systemctl).args(["restart", GRANTER_UNIT]))?;
ensure(&restart, "systemctl restart", &secrets)?;
Ok(Granted::Done)
let units = restart_failed_policy_units(tools, &secrets)?;
Ok(Granted { reapplied, units })
}
/// Resets and restarts every failed `swarm-bao-*-policy` unit. They retry on
/// their own, but one that hit its start limit stays down, and a granter that
/// was just re-applied is what lets them pass.
fn restart_failed_policy_units(tools: &Tools, secrets: &[&str]) -> Result<Vec<UnitRestart>> {
let listed = run(Command::new(&tools.systemctl).args([
"list-units",
"--state=failed",
"--plain",
"--no-legend",
"--no-pager",
"swarm-bao-*-policy.service",
]))?;
ensure(&listed, "systemctl list-units", secrets)?;
let mut units = Vec::new();
for name in String::from_utf8_lossy(&listed.stdout)
.lines()
.filter_map(|line| line.split_whitespace().next())
.filter(|name| name.ends_with(".service"))
{
let reset = run(Command::new(&tools.systemctl).args(["reset-failed", name]))?;
ensure(&reset, "systemctl reset-failed", secrets)?;
let restarted = run(Command::new(&tools.systemctl).args(["restart", name]))?;
let failure = ensure(&restarted, "systemctl restart", secrets)
.err()
.map(|e| format!("{e:#}"));
units.push(UnitRestart {
name: name.to_owned(),
failure,
});
}
Ok(units)
}
impl Tools {
@ -528,6 +586,9 @@ mod tests {
paths: GrantPaths,
}
/// The call that lists the failed policy units.
const LIST_FAILED: &str = "systemctl list-units --state=failed --plain --no-legend --no-pager swarm-bao-*-policy.service";
const STUB: &str = r#"#!/bin/sh
D=$(dirname "$0")
echo "ARGV $(basename "$0") $*" >> "$D/log"
@ -541,6 +602,12 @@ case "$(basename "$0") $1 $2 $3" in
cat "$D/policies"; exit 0 ;;
"bao policy write"*) exit 0 ;;
"bao token create"*) echo "hvs.GRANTTOKEN"; exit 0 ;;
"systemctl list-units --state=failed --plain")
cat "$D/failed_units" 2>/dev/null; exit 0 ;;
"systemctl reset-failed "*) exit 0 ;;
"systemctl restart swarm-bao-agent-ssh-policy.service"*)
if [ -e "$D/ssh_fails" ]; then echo "403 on sys/mounts/ssh-agents" >&2; exit 1; fi
exit 0 ;;
"systemctl restart "*)
if [ -e "$D/restart_fails" ]; then echo "unit failed, token file: $(cat "$D/grant.token")" >&2; exit 5; fi
exit 0 ;;
@ -694,7 +761,7 @@ exit 9
fn grant_runs_the_bootstrap_steps_in_order_with_no_token_in_any_argv() {
let f = Fixture::new("grant-order");
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
assert!(matches!(granted, Granted::Done));
assert!(!granted.reapplied);
let policy = f.paths.policy_file.display();
assert_eq!(
@ -706,6 +773,7 @@ exit 9
-display-name=bao-bootstrap -field=token"
.to_owned(),
"systemctl restart swarm-bao-granter-role".to_owned(),
LIST_FAILED.to_owned(),
]
);
let log = f.log();
@ -785,15 +853,87 @@ exit 9
);
}
/// The documented re-run after a policy change: the granter exists, and the
/// command still writes the policy, places the token and restarts the unit.
#[test]
fn grant_on_a_bootstrapped_store_changes_nothing() {
fn grant_on_a_bootstrapped_store_applies_the_policies_again() {
let f = Fixture::new("grant-again");
f.set("policies", "bao-granter\ndefault\nroot\n");
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("exits ok");
assert!(matches!(granted, Granted::AlreadyDone));
assert_eq!(f.calls(), ["bao policy list"]);
assert!(!f.paths.token_file.exists());
assert!(granted.describe(&f.paths).contains("already bootstrapped"));
assert!(granted.reapplied);
let policy = f.paths.policy_file.display();
assert_eq!(
f.calls(),
[
"bao policy list".to_owned(),
format!("bao policy write bao-bootstrap {policy}"),
"bao token create -policy=bao-bootstrap -ttl=24h -orphan \
-display-name=bao-bootstrap -field=token"
.to_owned(),
"systemctl restart swarm-bao-granter-role".to_owned(),
LIST_FAILED.to_owned(),
]
);
assert_eq!(
fs::read_to_string(&f.paths.token_file).expect("the token is placed"),
format!("{GRANT}\n")
);
let said = granted.describe(&f.paths);
assert!(said.contains("re-applied"), "{said}");
assert!(!said.contains("nothing done"), "{said}");
}
/// A unit that hit its start limit stays down on its own, so the command
/// resets and restarts each failed one, after the granter unit has run.
#[test]
fn failed_policy_units_are_reset_and_restarted_after_the_granter() {
let f = Fixture::new("grant-units");
f.set(
"failed_units",
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
swarm-bao-other-policy.service loaded failed failed other\n",
);
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
assert_eq!(
f.calls()[3..],
[
"systemctl restart swarm-bao-granter-role",
LIST_FAILED,
"systemctl reset-failed swarm-bao-agent-ssh-policy.service",
"systemctl restart swarm-bao-agent-ssh-policy.service",
"systemctl reset-failed swarm-bao-other-policy.service",
"systemctl restart swarm-bao-other-policy.service",
]
);
let said = granted.describe(&f.paths);
assert!(
said.contains("swarm-bao-agent-ssh-policy.service was failed and is active again"),
"{said}"
);
}
/// A unit that fails again is reported, and does not undo the granter's
/// success or stop the next unit.
#[test]
fn a_policy_unit_that_fails_again_is_reported_not_fatal() {
let f = Fixture::new("grant-units-again");
f.set(
"failed_units",
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
swarm-bao-other-policy.service loaded failed failed other\n",
);
f.set("ssh_fails", "");
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("the granter ran");
let said = granted.describe(&f.paths);
assert!(
said.contains("swarm-bao-agent-ssh-policy.service is still failing"),
"{said}"
);
assert!(said.contains("403"), "{said}");
assert!(
said.contains("swarm-bao-other-policy.service was failed and is active again"),
"{said}"
);
}
#[test]
@ -840,7 +980,7 @@ exit 9
);
assert_eq!(calls[3], "bao policy list");
assert_eq!(
calls.last().map(String::as_str),
calls.get(calls.len() - 2).map(String::as_str),
Some("systemctl restart swarm-bao-granter-role")
);
assert!(calls.iter().all(|c| !c.contains(ROOT)), "{calls:?}");