vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix(#970,#973): retry gateway reload on failure; always enable HIVE_WEB_SOCKET

Browse source
This commit is contained in:
damocles 2026-06-01 18:18:45 +02:00
commit b83edc40c6
5 changed files with 88 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

51
hive-c0re/src/gateway_nginx.rs
View file

@ -8,10 +8,18 @@
use anyhow::{Context, Result};
use std::fmt::Write as _;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::agent_sockets;
use crate::lifecycle;
/// Set when `write` publishes a new agents.conf; cleared when
/// `reload_gateway_nginx` submits the reload command successfully.
/// Lets `spawn_poll` retry the reload on subsequent ticks when the
/// previous attempt failed (e.g. gateway container temporarily down,
/// systemd-run not found) without re-writing the already-correct file.
static RELOAD_PENDING: AtomicBool = AtomicBool::new(false);
const HOST_CONF_PATH: &str = "/var/lib/hyperhive/gateway/agents.conf";
/// Host-side path where c0re writes the generated nginx include file.
@ -64,7 +72,7 @@ fn render(names: &[String], frontend_dir: Option<&str>) -> String {
let mut out = String::from(
"# Generated by hive-c0re \u{2014} do not edit.\
\n# Refreshed on every topology change + when agents bind/drop their unix sockets.\
\n# Gateway reloads nginx automatically on each update (systemd path unit).\n",
\n# Reload triggered by hive-c0re via systemd-run --machine=hive-gateway.\n",
);
for name in names {
let port = lifecycle::agent_web_port(name);
@ -184,24 +192,40 @@ pub fn write(names: &[String]) -> Result<()> {
path.display()
)
})?;
// Trigger nginx reload from the host. Ignore errors — a failed
// reload is recoverable (nginx keeps serving the previous config).
// Mark reload pending before attempting so a failed attempt is
// retried by the next spawn_poll tick (see `reload_if_pending`).
RELOAD_PENDING.store(true, Ordering::Relaxed);
reload_gateway_nginx();
Ok(())
}
/// Retry a pending nginx reload if a previous attempt failed.
/// Called by `spawn_poll` on each tick so a transient failure
/// (gateway container temporarily down, systemd-run error) is
/// recovered automatically without requiring a new file write.
pub fn reload_if_pending() {
if RELOAD_PENDING.load(Ordering::Relaxed) {
reload_gateway_nginx();
}
}
/// Send `nginx -s reload` to the gateway container via systemd-run.
/// Runs non-interactively in a transient scope so it doesn't block
/// c0re's polling loop. Best-effort: errors are logged, not bubbled.
/// Uses `--wait` so the exit code reflects whether nginx received the
/// signal; clears `RELOAD_PENDING` on success so `reload_if_pending`
/// stops retrying. Best-effort: errors are logged, not bubbled.
fn reload_gateway_nginx() {
// `--machine=hive-gateway` targets the container by its nspawn
// machine name (same as the nixos-container name). `--quiet`
// suppresses the transient unit name echo. `--` separates
// systemd-run args from the command.
// suppresses the transient unit name echo. `--wait` blocks until
// the transient job exits so the exit code tells us whether
// `nginx -s reload` ran at all (RELOAD_PENDING is only cleared on
// success — a failed attempt is retried next tick). `--`
// separates systemd-run args from the command.
let status = std::process::Command::new("systemd-run")
.args([
"--machine=hive-gateway",
"--quiet",
"--wait",
"--",
"nginx",
"-s",
@ -210,21 +234,22 @@ fn reload_gateway_nginx() {
.status();
match status {
Ok(s) if s.success() => {
// systemd-run accepted the request; nginx reload runs
// asynchronously inside the container and may still fail
// silently, but that's acceptable given the best-effort contract.
tracing::debug!("systemd-run accepted gateway nginx reload request");
// nginx -s reload ran successfully (SIGHUP sent to master).
// The actual worker replacement is async but the signal was
// delivered; clear the pending flag.
RELOAD_PENDING.store(false, Ordering::Relaxed);
tracing::debug!("gateway nginx reload signal sent");
}
Ok(s) => {
tracing::warn!(
exit_code = ?s.code(),
"gateway nginx reload exited non-zero; will pick up on next restart"
"gateway nginx reload exited non-zero — will retry next poll tick"
);
}
Err(e) => {
tracing::warn!(
error = %e,
"failed to invoke systemd-run for gateway nginx reload"
"failed to invoke systemd-run for gateway nginx reload — will retry next poll tick"
);
}
}