6a1e47532d
check_socket_activation and response_of are extracted from socket_listener and dispatch unchanged, and infra.rs takes a &impl Spawn, so tests run stand-ins instead of systemctl.
362 lines
13 KiB
Rust
362 lines
13 KiB
Rust
//! Host infrastructure ops: `ControlInfraContainer` (start/stop of an
|
|
//! [`InfraContainer`] unit) and `ReloadGatewayNginx`. Neither takes a unit or
|
|
//! container name from the caller.
|
|
|
|
use crate::shared::{RealSpawn, Spawn};
|
|
use anyhow::{Context as _, Result, bail};
|
|
use hive_priv_sock::{InfraAction, InfraContainer};
|
|
|
|
/// `ControlInfraContainer` — start/stop a hive infrastructure service via
|
|
/// `systemctl <verb> <unit>`. The [`InfraContainer`] enum is the allowlist:
|
|
/// serde already rejected any unknown / unsafe name (hive-c0re, the forge
|
|
/// and the matrix have no variant) at deserialisation, so no root-side
|
|
/// `.contains()` check is needed here. Serves the hive-wide
|
|
/// `hivectl stop`/`start`/`restart` flow. No agent-facing path exists.
|
|
///
|
|
/// ⚠️ The unit is derived from the variant, never sent by the caller —
|
|
/// which is what keeps this from being a general `systemctl` pass-through.
|
|
pub(crate) async fn control_infra_container(
|
|
container: InfraContainer,
|
|
action: InfraAction,
|
|
) -> Result<(String, String)> {
|
|
control_infra_container_with(&RealSpawn, container, action).await
|
|
}
|
|
|
|
async fn control_infra_container_with(
|
|
spawn: &impl Spawn,
|
|
container: InfraContainer,
|
|
action: InfraAction,
|
|
) -> Result<(String, String)> {
|
|
let verb = action.systemctl_verb();
|
|
let unit = container.service_unit();
|
|
let out = spawn
|
|
.command("systemctl")
|
|
.args([verb, unit])
|
|
.output()
|
|
.await
|
|
.with_context(|| format!("systemctl {verb} {unit}"))?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"systemctl {verb} {unit} exited {}: {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
tracing::info!(target: "infra-control", "{verb} {unit}");
|
|
Ok((
|
|
String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
String::from_utf8_lossy(&out.stderr).into_owned(),
|
|
))
|
|
}
|
|
|
|
/// Synchronise the host's nginx unit after an `agents.conf` write.
|
|
///
|
|
/// Queries `ActiveState` and dispatches:
|
|
/// - `active` → `systemctl reload nginx` (SIGHUP, zero-downtime)
|
|
/// - `failed` → `systemctl reset-failed nginx` + `systemctl start nginx`
|
|
/// - otherwise → `systemctl start nginx`
|
|
///
|
|
/// ⚠️ `nginx` is hard-coded on purpose — see `PrivRequest::ReloadGatewayNginx`.
|
|
/// The unit name is the scope of this verb: nginx is a host unit, so no
|
|
/// namespace bounds it and the literal is the only thing standing between
|
|
/// "reload the gateway" and "reload anything".
|
|
///
|
|
/// Returns `(String::new(), String::new())` on success so it fits the
|
|
/// `exec` return type directly.
|
|
pub(crate) async fn sync_gateway_nginx() -> Result<(String, String)> {
|
|
sync_gateway_nginx_with(&RealSpawn).await
|
|
}
|
|
|
|
async fn sync_gateway_nginx_with(spawn: &impl Spawn) -> Result<(String, String)> {
|
|
let state_out = spawn
|
|
.command("systemctl")
|
|
.args(["show", "--property=ActiveState", "--value", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("query gateway nginx ActiveState")?;
|
|
if !state_out.status.success() {
|
|
tracing::warn!(
|
|
exit_code = ?state_out.status.code(),
|
|
stderr = %String::from_utf8_lossy(&state_out.stderr).trim(),
|
|
"systemctl show ActiveState exited non-zero — gateway nginx may be down"
|
|
);
|
|
}
|
|
let state = String::from_utf8_lossy(&state_out.stdout).trim().to_owned();
|
|
// State-aware dispatch: reload when running; reset+start after
|
|
// start-limit failure; plain start when inactive or unknown.
|
|
match state.as_str() {
|
|
"active" => {
|
|
let out = spawn
|
|
.command("systemctl")
|
|
.args(["reload", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("reload gateway nginx")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx reload failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
"failed" => {
|
|
// Clear start-limit so the next start can proceed.
|
|
let _ = spawn
|
|
.command("systemctl")
|
|
.args(["reset-failed", "nginx"])
|
|
.status()
|
|
.await;
|
|
let out = spawn
|
|
.command("systemctl")
|
|
.args(["start", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("start gateway nginx after reset-failed")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx start (after reset-failed) failed ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
_ => {
|
|
// inactive, activating, deactivating, unknown — just start.
|
|
let out = spawn
|
|
.command("systemctl")
|
|
.args(["start", "nginx"])
|
|
.output()
|
|
.await
|
|
.context("start gateway nginx")?;
|
|
if !out.status.success() {
|
|
bail!(
|
|
"gateway nginx start failed (state={state:?}) ({}): {}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
);
|
|
}
|
|
}
|
|
}
|
|
Ok((String::new(), String::new()))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::path::PathBuf;
|
|
|
|
use super::{
|
|
InfraAction, InfraContainer, control_infra_container_with, sync_gateway_nginx_with,
|
|
};
|
|
use crate::shared::test_support::{MissingTool, ScriptSpawn, scratch};
|
|
use hive_priv_sock::PrivRequest;
|
|
|
|
/// A start/stop naming the forge or matrix container fails to decode, so
|
|
/// `run` answers it with a parse error before anything reaches
|
|
/// `systemctl`. Control: the same request naming the gateway decodes.
|
|
#[test]
|
|
fn infra_control_refuses_swarm_services() {
|
|
let req = |c: &str| {
|
|
format!(r#"{{"op":"control_infra_container","container":"{c}","action":"stop"}}"#)
|
|
};
|
|
for c in ["Forge", "Matrix"] {
|
|
let err = serde_json::from_str::<PrivRequest>(&req(c)).unwrap_err();
|
|
assert!(err.to_string().contains("unknown variant"), "{c}: {err}");
|
|
}
|
|
assert!(matches!(
|
|
serde_json::from_str::<PrivRequest>(&req("Gateway")),
|
|
Ok(PrivRequest::ControlInfraContainer {
|
|
container: InfraContainer::Gateway,
|
|
action: InfraAction::Stop,
|
|
})
|
|
));
|
|
}
|
|
|
|
/// Stand-in for `systemctl`: every call is logged as `systemctl args…`,
|
|
/// then `behave` (a shell script over `$@`) decides output and status.
|
|
struct Stub {
|
|
dir: PathBuf,
|
|
script: String,
|
|
}
|
|
|
|
impl Stub {
|
|
fn new(behave: &str) -> Self {
|
|
let dir = scratch();
|
|
let script = format!("echo \"$0 $*\" >> {d}/calls\n{behave}", d = dir.display());
|
|
Self { dir, script }
|
|
}
|
|
|
|
fn spawn(&self) -> ScriptSpawn {
|
|
ScriptSpawn(self.script.clone())
|
|
}
|
|
|
|
fn calls(&self) -> Vec<String> {
|
|
std::fs::read_to_string(self.dir.join("calls"))
|
|
.map(|log| log.lines().map(str::to_owned).collect())
|
|
.unwrap_or_default()
|
|
}
|
|
}
|
|
|
|
impl Drop for Stub {
|
|
fn drop(&mut self) {
|
|
std::fs::remove_dir_all(&self.dir).ok();
|
|
}
|
|
}
|
|
|
|
const SHOW: &str = "systemctl show --property=ActiveState --value nginx";
|
|
|
|
/// A host whose nginx reports `state`; every other verb succeeds.
|
|
fn nginx_in(state: &str) -> Stub {
|
|
Stub::new(&format!("case \"$1\" in show) echo {state};; esac"))
|
|
}
|
|
|
|
async fn sync(stub: &Stub) -> anyhow::Result<(String, String)> {
|
|
sync_gateway_nginx_with(&stub.spawn()).await
|
|
}
|
|
|
|
async fn control(
|
|
stub: &Stub,
|
|
container: InfraContainer,
|
|
action: InfraAction,
|
|
) -> anyhow::Result<(String, String)> {
|
|
control_infra_container_with(&stub.spawn(), container, action).await
|
|
}
|
|
|
|
fn err_text(r: anyhow::Result<(String, String)>) -> String {
|
|
format!("{:#}", r.unwrap_err())
|
|
}
|
|
|
|
// ---- ControlInfraContainer ----
|
|
|
|
/// The unit comes from the variant and the verb from the action: the
|
|
/// whole argv is `systemctl <verb> nginx.service`, nothing else.
|
|
#[tokio::test]
|
|
async fn control_runs_exactly_the_verb_on_the_variants_unit() {
|
|
for (action, verb) in [(InfraAction::Start, "start"), (InfraAction::Stop, "stop")] {
|
|
let stub = Stub::new("echo out; echo err >&2");
|
|
let (stdout, stderr) = control(&stub, InfraContainer::Gateway, action)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
stub.calls(),
|
|
[format!("systemctl {verb} nginx.service")],
|
|
"{verb}"
|
|
);
|
|
assert_eq!((stdout.as_str(), stderr.as_str()), ("out\n", "err\n"));
|
|
}
|
|
}
|
|
|
|
/// A non-zero exit is an error naming the verb, the unit, the status and
|
|
/// the trimmed stderr, so the operator sees why `hivectl` failed.
|
|
#[tokio::test]
|
|
async fn control_reports_a_failed_systemctl_with_its_stderr() {
|
|
let stub = Stub::new("echo 'unit is masked' >&2; exit 5");
|
|
let msg = err_text(control(&stub, InfraContainer::Gateway, InfraAction::Stop).await);
|
|
assert!(msg.contains("systemctl stop nginx.service exited"), "{msg}");
|
|
assert!(msg.ends_with("unit is masked"), "{msg}");
|
|
}
|
|
|
|
/// A host without `systemctl` is an error that names the command.
|
|
#[tokio::test]
|
|
async fn control_reports_a_missing_systemctl_by_name() {
|
|
let msg = err_text(
|
|
control_infra_container_with(&MissingTool, InfraContainer::Gateway, InfraAction::Start)
|
|
.await,
|
|
);
|
|
assert!(msg.contains("systemctl start nginx.service"), "{msg}");
|
|
}
|
|
|
|
// ---- ReloadGatewayNginx ----
|
|
|
|
/// A running nginx is reloaded, never restarted.
|
|
#[tokio::test]
|
|
async fn an_active_nginx_is_reloaded() {
|
|
let stub = nginx_in("active");
|
|
assert_eq!(sync(&stub).await.unwrap(), (String::new(), String::new()));
|
|
assert_eq!(stub.calls(), [SHOW, "systemctl reload nginx"]);
|
|
}
|
|
|
|
/// A failed nginx has its start-limit cleared before the start.
|
|
#[tokio::test]
|
|
async fn a_failed_nginx_is_reset_then_started() {
|
|
let stub = nginx_in("failed");
|
|
sync(&stub).await.unwrap();
|
|
assert_eq!(
|
|
stub.calls(),
|
|
[
|
|
SHOW,
|
|
"systemctl reset-failed nginx",
|
|
"systemctl start nginx"
|
|
]
|
|
);
|
|
}
|
|
|
|
/// Any other state, including one the query could not produce, is a
|
|
/// plain start: no reload of a unit that is not running.
|
|
#[tokio::test]
|
|
async fn every_other_state_is_a_plain_start() {
|
|
for state in ["inactive", "activating", "deactivating", "reloading", ""] {
|
|
let stub = nginx_in(state);
|
|
sync(&stub).await.unwrap();
|
|
assert_eq!(stub.calls(), [SHOW, "systemctl start nginx"], "{state:?}");
|
|
}
|
|
}
|
|
|
|
/// A failing state query is only logged: the sync still starts nginx
|
|
/// from whatever the query printed, here nothing.
|
|
#[tokio::test]
|
|
async fn a_failed_state_query_still_starts_nginx() {
|
|
let stub = Stub::new("case \"$1\" in show) echo boom >&2; exit 1;; esac");
|
|
sync(&stub).await.unwrap();
|
|
assert_eq!(stub.calls(), [SHOW, "systemctl start nginx"]);
|
|
}
|
|
|
|
/// A failed reload is an error carrying systemctl's stderr; a failed
|
|
/// `reset-failed` is not, because the start that follows is what counts.
|
|
#[tokio::test]
|
|
async fn a_failed_reload_is_an_error_and_a_failed_reset_is_not() {
|
|
let stub = Stub::new(
|
|
"case \"$1\" in show) echo active;; reload) echo 'bad config' >&2; exit 1;; esac",
|
|
);
|
|
let msg = err_text(sync(&stub).await);
|
|
assert!(msg.contains("gateway nginx reload failed"), "{msg}");
|
|
assert!(msg.ends_with("bad config"), "{msg}");
|
|
|
|
let stub = Stub::new("case \"$1\" in show) echo failed;; reset-failed) exit 1;; esac");
|
|
sync(&stub).await.unwrap();
|
|
assert_eq!(
|
|
stub.calls(),
|
|
[
|
|
SHOW,
|
|
"systemctl reset-failed nginx",
|
|
"systemctl start nginx"
|
|
]
|
|
);
|
|
}
|
|
|
|
/// A start that fails is an error on both start paths, and the plain
|
|
/// one names the state it started from.
|
|
#[tokio::test]
|
|
async fn a_failed_start_is_an_error_on_both_start_paths() {
|
|
let stub = Stub::new(
|
|
"case \"$1\" in show) echo failed;; start) echo 'start limit hit' >&2; exit 1;; esac",
|
|
);
|
|
let msg = err_text(sync(&stub).await);
|
|
assert!(msg.contains("start (after reset-failed) failed"), "{msg}");
|
|
assert!(msg.ends_with("start limit hit"), "{msg}");
|
|
|
|
let stub = Stub::new(
|
|
"case \"$1\" in show) echo inactive;; start) echo 'no such unit' >&2; exit 1;; esac",
|
|
);
|
|
let msg = err_text(sync(&stub).await);
|
|
assert!(msg.contains("state=\"inactive\""), "{msg}");
|
|
assert!(msg.ends_with("no such unit"), "{msg}");
|
|
}
|
|
|
|
/// Without `systemctl` the state query itself fails, and says so.
|
|
#[tokio::test]
|
|
async fn a_missing_systemctl_fails_the_state_query() {
|
|
let msg = err_text(sync_gateway_nginx_with(&MissingTool).await);
|
|
assert!(msg.contains("query gateway nginx ActiveState"), "{msg}");
|
|
}
|
|
}
|