grant-bootstrap re-applies the granter and restarts failed policy units; setup treats a refused granting unit as not done
This commit is contained in:
parent
ae37c228b1
commit
c7d37ca455
1 changed files with 271 additions and 33 deletions
|
|
@ -344,23 +344,47 @@ fn wait_unsealed(tools: &Tools) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// What `grant-bootstrap` did.
|
||||
#[derive(Debug)]
|
||||
enum Granted {
|
||||
AlreadyDone,
|
||||
Done,
|
||||
struct Granted {
|
||||
/// The granter already existed, so this re-applied the policies rather than
|
||||
/// bootstrapping for the first time.
|
||||
reapplied: bool,
|
||||
/// The failed `swarm-bao-*-policy` units that were reset and restarted
|
||||
/// once the granter unit had run.
|
||||
units: Vec<UnitRestart>,
|
||||
}
|
||||
|
||||
/// One failed policy unit and how its restart went.
|
||||
#[derive(Debug)]
|
||||
struct UnitRestart {
|
||||
name: String,
|
||||
/// Why it failed again, `None` when it is active now.
|
||||
failure: Option<String>,
|
||||
}
|
||||
|
||||
impl Granted {
|
||||
fn describe(&self, paths: &GrantPaths) -> String {
|
||||
match self {
|
||||
Self::AlreadyDone => format!(
|
||||
"the granter is already bootstrapped (policy {GRANTER_POLICY} exists); nothing done"
|
||||
),
|
||||
Self::Done => format!(
|
||||
let first = if self.reapplied {
|
||||
format!(
|
||||
"granter re-applied: {BOOTSTRAP_POLICY} written, grant token written to {}, \
|
||||
{GRANTER_UNIT} restarted to rewrite {GRANTER_POLICY} from the deployed policy",
|
||||
paths.token_file.display()
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"granter bootstrap started: grant token written to {}, {GRANTER_UNIT} restarted",
|
||||
paths.token_file.display()
|
||||
),
|
||||
}
|
||||
)
|
||||
};
|
||||
let units = self.units.iter().map(|unit| match &unit.failure {
|
||||
None => format!("{} was failed and is active again", unit.name),
|
||||
Some(why) => format!("{} is still failing: {why}", unit.name),
|
||||
});
|
||||
std::iter::once(first)
|
||||
.chain(units)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -370,12 +394,12 @@ impl Granted {
|
|||
fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result<Granted> {
|
||||
let policies = run(tools.bao_cmd(&["policy", "list"]).env("BAO_TOKEN", root))?;
|
||||
ensure(&policies, "bao policy list", &[root])?;
|
||||
if String::from_utf8_lossy(&policies.stdout)
|
||||
// A granter that exists is not a reason to stop: the deployed policy text
|
||||
// may be newer than the one in the store, and the granter unit is what
|
||||
// writes it. So the same steps run again.
|
||||
let reapplied = String::from_utf8_lossy(&policies.stdout)
|
||||
.lines()
|
||||
.any(|line| line.trim() == GRANTER_POLICY)
|
||||
{
|
||||
return Ok(Granted::AlreadyDone);
|
||||
}
|
||||
.any(|line| line.trim() == GRANTER_POLICY);
|
||||
|
||||
let write = run(tools
|
||||
.bao_cmd(&["policy", "write", BOOTSTRAP_POLICY])
|
||||
|
|
@ -422,7 +446,41 @@ fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result
|
|||
|
||||
let restart = run(Command::new(&tools.systemctl).args(["restart", GRANTER_UNIT]))?;
|
||||
ensure(&restart, "systemctl restart", &secrets)?;
|
||||
Ok(Granted::Done)
|
||||
let units = restart_failed_policy_units(tools, &secrets)?;
|
||||
Ok(Granted { reapplied, units })
|
||||
}
|
||||
|
||||
/// Resets and restarts every failed `swarm-bao-*-policy` unit. They retry on
|
||||
/// their own, but one that hit its start limit stays down, and a granter that
|
||||
/// was just re-applied is what lets them pass.
|
||||
fn restart_failed_policy_units(tools: &Tools, secrets: &[&str]) -> Result<Vec<UnitRestart>> {
|
||||
let listed = run(Command::new(&tools.systemctl).args([
|
||||
"list-units",
|
||||
"--state=failed",
|
||||
"--plain",
|
||||
"--no-legend",
|
||||
"--no-pager",
|
||||
"swarm-bao-*-policy.service",
|
||||
]))?;
|
||||
ensure(&listed, "systemctl list-units", secrets)?;
|
||||
let mut units = Vec::new();
|
||||
for name in String::from_utf8_lossy(&listed.stdout)
|
||||
.lines()
|
||||
.filter_map(|line| line.split_whitespace().next())
|
||||
.filter(|name| name.ends_with(".service"))
|
||||
{
|
||||
let reset = run(Command::new(&tools.systemctl).args(["reset-failed", name]))?;
|
||||
ensure(&reset, "systemctl reset-failed", secrets)?;
|
||||
let restarted = run(Command::new(&tools.systemctl).args(["restart", name]))?;
|
||||
let failure = ensure(&restarted, "systemctl restart", secrets)
|
||||
.err()
|
||||
.map(|e| format!("{e:#}"));
|
||||
units.push(UnitRestart {
|
||||
name: name.to_owned(),
|
||||
failure,
|
||||
});
|
||||
}
|
||||
Ok(units)
|
||||
}
|
||||
|
||||
impl Tools {
|
||||
|
|
@ -528,6 +586,9 @@ mod tests {
|
|||
paths: GrantPaths,
|
||||
}
|
||||
|
||||
/// The call that lists the failed policy units.
|
||||
const LIST_FAILED: &str = "systemctl list-units --state=failed --plain --no-legend --no-pager swarm-bao-*-policy.service";
|
||||
|
||||
const STUB: &str = r#"#!/bin/sh
|
||||
D=$(dirname "$0")
|
||||
echo "ARGV $(basename "$0") $*" >> "$D/log"
|
||||
|
|
@ -541,6 +602,12 @@ case "$(basename "$0") $1 $2 $3" in
|
|||
cat "$D/policies"; exit 0 ;;
|
||||
"bao policy write"*) exit 0 ;;
|
||||
"bao token create"*) echo "hvs.GRANTTOKEN"; exit 0 ;;
|
||||
"systemctl list-units --state=failed --plain")
|
||||
cat "$D/failed_units" 2>/dev/null; exit 0 ;;
|
||||
"systemctl reset-failed "*) exit 0 ;;
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service"*)
|
||||
if [ -e "$D/ssh_fails" ]; then echo "403 on sys/mounts/ssh-agents" >&2; exit 1; fi
|
||||
exit 0 ;;
|
||||
"systemctl restart "*)
|
||||
if [ -e "$D/restart_fails" ]; then echo "unit failed, token file: $(cat "$D/grant.token")" >&2; exit 5; fi
|
||||
exit 0 ;;
|
||||
|
|
@ -694,7 +761,7 @@ exit 9
|
|||
fn grant_runs_the_bootstrap_steps_in_order_with_no_token_in_any_argv() {
|
||||
let f = Fixture::new("grant-order");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert!(matches!(granted, Granted::Done));
|
||||
assert!(!granted.reapplied);
|
||||
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
|
|
@ -706,6 +773,7 @@ exit 9
|
|||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
let log = f.log();
|
||||
|
|
@ -785,15 +853,87 @@ exit 9
|
|||
);
|
||||
}
|
||||
|
||||
/// The documented re-run after a policy change: the granter exists, and the
|
||||
/// command still writes the policy, places the token and restarts the unit.
|
||||
#[test]
|
||||
fn grant_on_a_bootstrapped_store_changes_nothing() {
|
||||
fn grant_on_a_bootstrapped_store_applies_the_policies_again() {
|
||||
let f = Fixture::new("grant-again");
|
||||
f.set("policies", "bao-granter\ndefault\nroot\n");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("exits ok");
|
||||
assert!(matches!(granted, Granted::AlreadyDone));
|
||||
assert_eq!(f.calls(), ["bao policy list"]);
|
||||
assert!(!f.paths.token_file.exists());
|
||||
assert!(granted.describe(&f.paths).contains("already bootstrapped"));
|
||||
assert!(granted.reapplied);
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
f.calls(),
|
||||
[
|
||||
"bao policy list".to_owned(),
|
||||
format!("bao policy write bao-bootstrap {policy}"),
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan \
|
||||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(&f.paths.token_file).expect("the token is placed"),
|
||||
format!("{GRANT}\n")
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(said.contains("re-applied"), "{said}");
|
||||
assert!(!said.contains("nothing done"), "{said}");
|
||||
}
|
||||
|
||||
/// A unit that hit its start limit stays down on its own, so the command
|
||||
/// resets and restarts each failed one, after the granter unit has run.
|
||||
#[test]
|
||||
fn failed_policy_units_are_reset_and_restarted_after_the_granter() {
|
||||
let f = Fixture::new("grant-units");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert_eq!(
|
||||
f.calls()[3..],
|
||||
[
|
||||
"systemctl restart swarm-bao-granter-role",
|
||||
LIST_FAILED,
|
||||
"systemctl reset-failed swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl reset-failed swarm-bao-other-policy.service",
|
||||
"systemctl restart swarm-bao-other-policy.service",
|
||||
]
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A unit that fails again is reported, and does not undo the granter's
|
||||
/// success or stop the next unit.
|
||||
#[test]
|
||||
fn a_policy_unit_that_fails_again_is_reported_not_fatal() {
|
||||
let f = Fixture::new("grant-units-again");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
f.set("ssh_fails", "");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("the granter ran");
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service is still failing"),
|
||||
"{said}"
|
||||
);
|
||||
assert!(said.contains("403"), "{said}");
|
||||
assert!(
|
||||
said.contains("swarm-bao-other-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -840,7 +980,7 @@ exit 9
|
|||
);
|
||||
assert_eq!(calls[3], "bao policy list");
|
||||
assert_eq!(
|
||||
calls.last().map(String::as_str),
|
||||
calls.get(calls.len() - 2).map(String::as_str),
|
||||
Some("systemctl restart swarm-bao-granter-role")
|
||||
);
|
||||
assert!(calls.iter().all(|c| !c.contains(ROOT)), "{calls:?}");
|
||||
|
|
|
|||
Loading…
Reference in a new issue