grant-bootstrap re-applies the granter and restarts failed policy units; setup treats a refused granting unit as not done
This commit is contained in:
parent
ae37c228b1
commit
c7d37ca455
4 changed files with 271 additions and 33 deletions
|
|
@ -344,23 +344,47 @@ fn wait_unsealed(tools: &Tools) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
/// What `grant-bootstrap` did.
|
||||
#[derive(Debug)]
|
||||
enum Granted {
|
||||
AlreadyDone,
|
||||
Done,
|
||||
struct Granted {
|
||||
/// The granter already existed, so this re-applied the policies rather than
|
||||
/// bootstrapping for the first time.
|
||||
reapplied: bool,
|
||||
/// The failed `swarm-bao-*-policy` units that were reset and restarted
|
||||
/// once the granter unit had run.
|
||||
units: Vec<UnitRestart>,
|
||||
}
|
||||
|
||||
/// One failed policy unit and how its restart went.
|
||||
#[derive(Debug)]
|
||||
struct UnitRestart {
|
||||
name: String,
|
||||
/// Why it failed again, `None` when it is active now.
|
||||
failure: Option<String>,
|
||||
}
|
||||
|
||||
impl Granted {
|
||||
fn describe(&self, paths: &GrantPaths) -> String {
|
||||
match self {
|
||||
Self::AlreadyDone => format!(
|
||||
"the granter is already bootstrapped (policy {GRANTER_POLICY} exists); nothing done"
|
||||
),
|
||||
Self::Done => format!(
|
||||
let first = if self.reapplied {
|
||||
format!(
|
||||
"granter re-applied: {BOOTSTRAP_POLICY} written, grant token written to {}, \
|
||||
{GRANTER_UNIT} restarted to rewrite {GRANTER_POLICY} from the deployed policy",
|
||||
paths.token_file.display()
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"granter bootstrap started: grant token written to {}, {GRANTER_UNIT} restarted",
|
||||
paths.token_file.display()
|
||||
),
|
||||
}
|
||||
)
|
||||
};
|
||||
let units = self.units.iter().map(|unit| match &unit.failure {
|
||||
None => format!("{} was failed and is active again", unit.name),
|
||||
Some(why) => format!("{} is still failing: {why}", unit.name),
|
||||
});
|
||||
std::iter::once(first)
|
||||
.chain(units)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -370,12 +394,12 @@ impl Granted {
|
|||
fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result<Granted> {
|
||||
let policies = run(tools.bao_cmd(&["policy", "list"]).env("BAO_TOKEN", root))?;
|
||||
ensure(&policies, "bao policy list", &[root])?;
|
||||
if String::from_utf8_lossy(&policies.stdout)
|
||||
// A granter that exists is not a reason to stop: the deployed policy text
|
||||
// may be newer than the one in the store, and the granter unit is what
|
||||
// writes it. So the same steps run again.
|
||||
let reapplied = String::from_utf8_lossy(&policies.stdout)
|
||||
.lines()
|
||||
.any(|line| line.trim() == GRANTER_POLICY)
|
||||
{
|
||||
return Ok(Granted::AlreadyDone);
|
||||
}
|
||||
.any(|line| line.trim() == GRANTER_POLICY);
|
||||
|
||||
let write = run(tools
|
||||
.bao_cmd(&["policy", "write", BOOTSTRAP_POLICY])
|
||||
|
|
@ -422,7 +446,41 @@ fn grant_bootstrap_with(tools: &Tools, paths: &GrantPaths, root: &str) -> Result
|
|||
|
||||
let restart = run(Command::new(&tools.systemctl).args(["restart", GRANTER_UNIT]))?;
|
||||
ensure(&restart, "systemctl restart", &secrets)?;
|
||||
Ok(Granted::Done)
|
||||
let units = restart_failed_policy_units(tools, &secrets)?;
|
||||
Ok(Granted { reapplied, units })
|
||||
}
|
||||
|
||||
/// Resets and restarts every failed `swarm-bao-*-policy` unit. They retry on
|
||||
/// their own, but one that hit its start limit stays down, and a granter that
|
||||
/// was just re-applied is what lets them pass.
|
||||
fn restart_failed_policy_units(tools: &Tools, secrets: &[&str]) -> Result<Vec<UnitRestart>> {
|
||||
let listed = run(Command::new(&tools.systemctl).args([
|
||||
"list-units",
|
||||
"--state=failed",
|
||||
"--plain",
|
||||
"--no-legend",
|
||||
"--no-pager",
|
||||
"swarm-bao-*-policy.service",
|
||||
]))?;
|
||||
ensure(&listed, "systemctl list-units", secrets)?;
|
||||
let mut units = Vec::new();
|
||||
for name in String::from_utf8_lossy(&listed.stdout)
|
||||
.lines()
|
||||
.filter_map(|line| line.split_whitespace().next())
|
||||
.filter(|name| name.ends_with(".service"))
|
||||
{
|
||||
let reset = run(Command::new(&tools.systemctl).args(["reset-failed", name]))?;
|
||||
ensure(&reset, "systemctl reset-failed", secrets)?;
|
||||
let restarted = run(Command::new(&tools.systemctl).args(["restart", name]))?;
|
||||
let failure = ensure(&restarted, "systemctl restart", secrets)
|
||||
.err()
|
||||
.map(|e| format!("{e:#}"));
|
||||
units.push(UnitRestart {
|
||||
name: name.to_owned(),
|
||||
failure,
|
||||
});
|
||||
}
|
||||
Ok(units)
|
||||
}
|
||||
|
||||
impl Tools {
|
||||
|
|
@ -528,6 +586,9 @@ mod tests {
|
|||
paths: GrantPaths,
|
||||
}
|
||||
|
||||
/// The call that lists the failed policy units.
|
||||
const LIST_FAILED: &str = "systemctl list-units --state=failed --plain --no-legend --no-pager swarm-bao-*-policy.service";
|
||||
|
||||
const STUB: &str = r#"#!/bin/sh
|
||||
D=$(dirname "$0")
|
||||
echo "ARGV $(basename "$0") $*" >> "$D/log"
|
||||
|
|
@ -541,6 +602,12 @@ case "$(basename "$0") $1 $2 $3" in
|
|||
cat "$D/policies"; exit 0 ;;
|
||||
"bao policy write"*) exit 0 ;;
|
||||
"bao token create"*) echo "hvs.GRANTTOKEN"; exit 0 ;;
|
||||
"systemctl list-units --state=failed --plain")
|
||||
cat "$D/failed_units" 2>/dev/null; exit 0 ;;
|
||||
"systemctl reset-failed "*) exit 0 ;;
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service"*)
|
||||
if [ -e "$D/ssh_fails" ]; then echo "403 on sys/mounts/ssh-agents" >&2; exit 1; fi
|
||||
exit 0 ;;
|
||||
"systemctl restart "*)
|
||||
if [ -e "$D/restart_fails" ]; then echo "unit failed, token file: $(cat "$D/grant.token")" >&2; exit 5; fi
|
||||
exit 0 ;;
|
||||
|
|
@ -694,7 +761,7 @@ exit 9
|
|||
fn grant_runs_the_bootstrap_steps_in_order_with_no_token_in_any_argv() {
|
||||
let f = Fixture::new("grant-order");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert!(matches!(granted, Granted::Done));
|
||||
assert!(!granted.reapplied);
|
||||
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
|
|
@ -706,6 +773,7 @@ exit 9
|
|||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
let log = f.log();
|
||||
|
|
@ -785,15 +853,87 @@ exit 9
|
|||
);
|
||||
}
|
||||
|
||||
/// The documented re-run after a policy change: the granter exists, and the
|
||||
/// command still writes the policy, places the token and restarts the unit.
|
||||
#[test]
|
||||
fn grant_on_a_bootstrapped_store_changes_nothing() {
|
||||
fn grant_on_a_bootstrapped_store_applies_the_policies_again() {
|
||||
let f = Fixture::new("grant-again");
|
||||
f.set("policies", "bao-granter\ndefault\nroot\n");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("exits ok");
|
||||
assert!(matches!(granted, Granted::AlreadyDone));
|
||||
assert_eq!(f.calls(), ["bao policy list"]);
|
||||
assert!(!f.paths.token_file.exists());
|
||||
assert!(granted.describe(&f.paths).contains("already bootstrapped"));
|
||||
assert!(granted.reapplied);
|
||||
let policy = f.paths.policy_file.display();
|
||||
assert_eq!(
|
||||
f.calls(),
|
||||
[
|
||||
"bao policy list".to_owned(),
|
||||
format!("bao policy write bao-bootstrap {policy}"),
|
||||
"bao token create -policy=bao-bootstrap -ttl=24h -orphan \
|
||||
-display-name=bao-bootstrap -field=token"
|
||||
.to_owned(),
|
||||
"systemctl restart swarm-bao-granter-role".to_owned(),
|
||||
LIST_FAILED.to_owned(),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(&f.paths.token_file).expect("the token is placed"),
|
||||
format!("{GRANT}\n")
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(said.contains("re-applied"), "{said}");
|
||||
assert!(!said.contains("nothing done"), "{said}");
|
||||
}
|
||||
|
||||
/// A unit that hit its start limit stays down on its own, so the command
|
||||
/// resets and restarts each failed one, after the granter unit has run.
|
||||
#[test]
|
||||
fn failed_policy_units_are_reset_and_restarted_after_the_granter() {
|
||||
let f = Fixture::new("grant-units");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("grant succeeds");
|
||||
assert_eq!(
|
||||
f.calls()[3..],
|
||||
[
|
||||
"systemctl restart swarm-bao-granter-role",
|
||||
LIST_FAILED,
|
||||
"systemctl reset-failed swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl restart swarm-bao-agent-ssh-policy.service",
|
||||
"systemctl reset-failed swarm-bao-other-policy.service",
|
||||
"systemctl restart swarm-bao-other-policy.service",
|
||||
]
|
||||
);
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A unit that fails again is reported, and does not undo the granter's
|
||||
/// success or stop the next unit.
|
||||
#[test]
|
||||
fn a_policy_unit_that_fails_again_is_reported_not_fatal() {
|
||||
let f = Fixture::new("grant-units-again");
|
||||
f.set(
|
||||
"failed_units",
|
||||
"swarm-bao-agent-ssh-policy.service loaded failed failed ssh\n\
|
||||
swarm-bao-other-policy.service loaded failed failed other\n",
|
||||
);
|
||||
f.set("ssh_fails", "");
|
||||
let granted = grant_bootstrap_with(&f.tools, &f.paths, ROOT).expect("the granter ran");
|
||||
let said = granted.describe(&f.paths);
|
||||
assert!(
|
||||
said.contains("swarm-bao-agent-ssh-policy.service is still failing"),
|
||||
"{said}"
|
||||
);
|
||||
assert!(said.contains("403"), "{said}");
|
||||
assert!(
|
||||
said.contains("swarm-bao-other-policy.service was failed and is active again"),
|
||||
"{said}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -840,7 +980,7 @@ exit 9
|
|||
);
|
||||
assert_eq!(calls[3], "bao policy list");
|
||||
assert_eq!(
|
||||
calls.last().map(String::as_str),
|
||||
calls.get(calls.len() - 2).map(String::as_str),
|
||||
Some("systemctl restart swarm-bao-granter-role")
|
||||
);
|
||||
assert!(calls.iter().all(|c| !c.contains(ROOT)), "{calls:?}");
|
||||
|
|
|
|||
|
|
@ -842,6 +842,7 @@ mod tests {
|
|||
"failed"
|
||||
}
|
||||
.to_owned(),
|
||||
denied: false,
|
||||
}]),
|
||||
admin_user: Ok(self.user),
|
||||
agents: Ok(Agents {
|
||||
|
|
|
|||
|
|
@ -333,8 +333,10 @@ enum BaoVerb {
|
|||
/// Reads the root token from a hidden prompt, or from stdin when stdin isn't
|
||||
/// a terminal; never from an argument or an environment variable. Writes
|
||||
/// the `bao-bootstrap` policy, places a 24h grant token for the granter unit
|
||||
/// and restarts it, so run it as root on the host running that unit. Says so
|
||||
/// and does nothing when the granter exists already.
|
||||
/// and restarts it, so run it as root on the host running that unit. When
|
||||
/// the granter exists already it runs the same steps again, which is how a
|
||||
/// change to the granter's policy reaches an existing store, then resets
|
||||
/// and restarts the failed `swarm-bao-*-policy` units.
|
||||
GrantBootstrap(bao::GrantBootstrapArgs),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,10 @@ pub(crate) struct Granter {
|
|||
pub(crate) struct Unit {
|
||||
pub(crate) name: String,
|
||||
pub(crate) state: String,
|
||||
/// The unit failed and its journal's last lines show the store refusing
|
||||
/// it with a 403 or "permission denied": the granter's policy in the store
|
||||
/// is older than the deployed one.
|
||||
pub(crate) denied: bool,
|
||||
}
|
||||
|
||||
/// One row of `GET /api/agents/status`, the two fields read.
|
||||
|
|
@ -165,7 +169,7 @@ const STEPS: [(&str, &str, bool); 10] = [
|
|||
pub(crate) fn classify(observed: &Observed) -> Vec<Step> {
|
||||
let unsealed = unsealed_status(&observed.store);
|
||||
let policy_units = policy_units_status(&observed.policy_units);
|
||||
let granter = granter_status(&observed.granter, &policy_units);
|
||||
let granter = granter_status(&observed.granter, &observed.policy_units, &policy_units);
|
||||
let reunsealed = match (&unsealed, &policy_units) {
|
||||
(Status::Done(_), Status::Done(_)) => Status::Done(String::new()),
|
||||
(Status::Unknown(e), _) | (_, Status::Unknown(e)) => Status::Unknown(e.clone()),
|
||||
|
|
@ -254,7 +258,23 @@ fn policy_units_status(units: &Result<Vec<Unit>, String>) -> Status {
|
|||
/// the token is absent, so after a reboot it is `inactive` whether or not it
|
||||
/// ever ran. An active granting unit logs in through the granter, so that
|
||||
/// also proves the granter exists.
|
||||
fn granter_status(granter: &Result<Granter, String>, policy_units: &Status) -> Status {
|
||||
fn granter_status(
|
||||
granter: &Result<Granter, String>,
|
||||
units: &Result<Vec<Unit>, String>,
|
||||
policy_units: &Status,
|
||||
) -> Status {
|
||||
// A granting unit the store refuses means the granter exists but is
|
||||
// behind: its upload in the journal and an active unit of its own say
|
||||
// nothing about the policy the store holds now.
|
||||
if let Ok(units) = units
|
||||
&& let Some(unit) = units.iter().find(|u| u.denied)
|
||||
{
|
||||
return Status::NotDone(format!(
|
||||
"{} was refused by the store (403): its granter policy is older than the deployed \
|
||||
one; `swarmctl bao grant-bootstrap` applies it",
|
||||
unit.name
|
||||
));
|
||||
}
|
||||
if matches!(policy_units, Status::Done(_)) {
|
||||
return Status::Done("the granting units log in through it".to_owned());
|
||||
}
|
||||
|
|
@ -655,13 +675,36 @@ fn policy_units() -> Result<Vec<Unit>, String> {
|
|||
Ok(names
|
||||
.into_iter()
|
||||
.zip(states)
|
||||
.map(|(name, state)| Unit {
|
||||
name,
|
||||
state: state.to_owned(),
|
||||
.map(|(name, state)| {
|
||||
let denied = state == "failed" && last_error_is_denied(&name);
|
||||
Unit {
|
||||
name,
|
||||
state: state.to_owned(),
|
||||
denied,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// How many of a failed unit's last journal lines are read for its error.
|
||||
const JOURNAL_TAIL: &str = "30";
|
||||
|
||||
/// Whether the unit's last journal lines show the store refusing it. An
|
||||
/// unreadable journal counts as no: the unit's state decides.
|
||||
fn last_error_is_denied(unit: &str) -> bool {
|
||||
run(Command::new("journalctl").args(["-u", unit, "-n", JOURNAL_TAIL, "--no-pager", "-q"]))
|
||||
.is_ok_and(|j| journal_shows_denial(&j.stdout))
|
||||
}
|
||||
|
||||
/// A 403 or "permission denied" in `journal`, as the `bao` CLI prints a refusal
|
||||
/// (`Code: 403` and `permission denied`).
|
||||
fn journal_shows_denial(journal: &str) -> bool {
|
||||
journal.lines().any(|line| {
|
||||
let line = line.to_ascii_lowercase();
|
||||
line.contains("permission denied") || line.contains("code: 403")
|
||||
})
|
||||
}
|
||||
|
||||
/// The unit names in `systemctl list-unit-files --no-legend`, without the one
|
||||
/// that waits on authelia.
|
||||
fn parse_unit_files(listed: &str) -> Vec<String> {
|
||||
|
|
@ -742,6 +785,7 @@ mod tests {
|
|||
.map(|(i, s)| Unit {
|
||||
name: format!("swarm-bao-u{i}-policy.service"),
|
||||
state: (*s).to_owned(),
|
||||
denied: false,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
@ -882,6 +926,45 @@ mod tests {
|
|||
assert_eq!(kind(&status_of(&o, 3)), "done");
|
||||
}
|
||||
|
||||
/// A granting unit the store refuses with a 403 means the granter exists but
|
||||
/// its policy in the store is behind the deployed one: an upload in its
|
||||
/// journal and an active granter unit don't make step 3 done.
|
||||
#[test]
|
||||
fn a_granting_unit_refused_with_a_403_means_the_granter_is_not_done() {
|
||||
let mut o = finished();
|
||||
o.granter = Ok(Granter {
|
||||
state: "active".to_owned(),
|
||||
uploaded: true,
|
||||
});
|
||||
o.policy_units = Ok(units(&["active", "failed"]));
|
||||
// The control: a failed unit that is not a refusal leaves the granter done.
|
||||
assert_eq!(kind(&status_of(&o, 3)), "done");
|
||||
if let Ok(units) = &mut o.policy_units {
|
||||
units[1].denied = true;
|
||||
}
|
||||
assert!(matches!(
|
||||
status_of(&o, 3),
|
||||
Status::NotDone(why) if why.contains("u1-policy") && why.contains("swarmctl bao grant-bootstrap")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refusal_is_read_from_the_journal_as_the_bao_cli_prints_it() {
|
||||
let refused = "Error enabling: Error making API request.\n\n\
|
||||
URL: POST https://127.0.0.1:8200/v1/sys/mounts/ssh-agents\n\
|
||||
Code: 403. Errors:\n\n* 1 error occurred:\n\t* permission denied\n";
|
||||
assert!(journal_shows_denial(refused));
|
||||
assert!(journal_shows_denial("something\nCode: 403. Errors:"));
|
||||
// The controls: other failures, and nothing at all.
|
||||
assert!(!journal_shows_denial(
|
||||
"dial tcp 127.0.0.1:8200: connection refused\n"
|
||||
));
|
||||
assert!(!journal_shows_denial(
|
||||
"the store is sealed or unreachable; retrying.\n"
|
||||
));
|
||||
assert!(!journal_shows_denial(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_inactive_policy_unit_fails_step_four_and_names_it() {
|
||||
let mut o = finished();
|
||||
|
|
|
|||
Loading…
Reference in a new issue