Compare commits
6 changed files with 226 additions and 263 deletions
|
|
@ -1,116 +0,0 @@
|
||||||
//! hive-ci Forgejo Actions runner registration, driven from hive-c0re.
|
|
||||||
//!
|
|
||||||
//! Moves the runner-registration forge round-trip OFF the container's
|
|
||||||
//! boot-critical path (it was a host-side `hive-ci-prefetch` oneshot that
|
|
||||||
//! gated `container@hive-ci` start — see `nix/host-modules/hive-ci.nix`).
|
|
||||||
//! hive-c0re holds the forge admin token; here it validates the runner's
|
|
||||||
//! existing credentials and, when they're absent or stale, mints a fresh
|
|
||||||
//! registration token and hands it to hive-priv, which writes it to the host
|
|
||||||
//! env-file the container bind-mounts read-only and restarts the in-container
|
|
||||||
//! runner. The admin token never enters the container — only the registration
|
|
||||||
//! token does, exactly as the prefetch did.
|
|
||||||
|
|
||||||
use anyhow::{Context as _, Result};
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
use super::forge_http_base;
|
|
||||||
|
|
||||||
/// Host path to the hive-ci runner's persisted credentials. The container is
|
|
||||||
/// non-ephemeral, so `.runner` survives restarts; a present file with a sane
|
|
||||||
/// `id` means the runner is already registered.
|
|
||||||
const RUNNER_FILE: &str = "/var/lib/nixos-containers/hive-ci/var/lib/gitea-runner/hive/.runner";
|
|
||||||
|
|
||||||
/// Whether the operator enabled the CI runner. The nix module sets
|
|
||||||
/// `HYPERHIVE_FORGE_CI_ENABLED=1` on `hive-c0re.service` when
|
|
||||||
/// `services.hyperhive.forge.ci.enable` is on; absent means CI is off and
|
|
||||||
/// there is no hive-ci container to register a runner for.
|
|
||||||
fn ci_enabled() -> bool {
|
|
||||||
std::env::var("HYPERHIVE_FORGE_CI_ENABLED")
|
|
||||||
.is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ensure the hive-ci runner is registered against the forge. Best-effort:
|
|
||||||
/// every failure is logged and swallowed so it never aborts the startup
|
|
||||||
/// sweep ([`super::ensure_all`]). No-op when CI is disabled or the runner
|
|
||||||
/// already holds valid credentials (so a healthy runner is never restarted).
|
|
||||||
pub(super) async fn ensure_ci_runner_registered(core_token: &str) {
|
|
||||||
if !ci_enabled() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Existing creds still valid on the forge → nothing to do.
|
|
||||||
if let Some(id) = existing_runner_id()
|
|
||||||
&& runner_valid(core_token, id).await
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Absent or stale → mint a fresh registration token and hand it to
|
|
||||||
// hive-priv (root) to write the env-file + restart the runner.
|
|
||||||
match fetch_registration_token(core_token).await {
|
|
||||||
Ok(token) => {
|
|
||||||
if let Err(e) = crate::priv_client::register_ci_runner(&token).await {
|
|
||||||
tracing::warn!(error = ?e, "ci runner: hive-priv register_ci_runner failed");
|
|
||||||
} else {
|
|
||||||
tracing::info!("ci runner: registered hive-ci runner with a fresh token");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(e) => tracing::warn!(error = ?e, "ci runner: fetch registration token failed"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse the runner id from the persisted `.runner` JSON, if present and sane
|
|
||||||
/// (a `0` id is the gitea-runner "unregistered" sentinel).
|
|
||||||
fn existing_runner_id() -> Option<u64> {
|
|
||||||
let raw = std::fs::read_to_string(RUNNER_FILE).ok()?;
|
|
||||||
let json: Value = serde_json::from_str(&raw).ok()?;
|
|
||||||
let id = json.get("id")?.as_u64()?;
|
|
||||||
(id != 0).then_some(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `GET /admin/runners/{id}` — `true` iff the runner still exists on the forge
|
|
||||||
/// (HTTP 200). A 404 (deleted from the admin panel) or any other status means
|
|
||||||
/// re-registration is needed. A transport error (forge unreachable) is treated
|
|
||||||
/// as "keep the existing creds" so a network blip never wipes a valid runner.
|
|
||||||
async fn runner_valid(core_token: &str, id: u64) -> bool {
|
|
||||||
let url = format!("{}/api/v1/admin/runners/{id}", forge_http_base());
|
|
||||||
match reqwest::Client::new()
|
|
||||||
.get(&url)
|
|
||||||
.header("Authorization", format!("token {core_token}"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(resp) => resp.status().is_success(),
|
|
||||||
Err(e) => {
|
|
||||||
tracing::warn!(error = ?e, "ci runner: validation request failed; keeping existing creds");
|
|
||||||
true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `GET /admin/runners/registration-token` — mint a fresh registration token.
|
|
||||||
/// forgejo-api (0.11) doesn't wrap this endpoint, so this is a raw request
|
|
||||||
/// against the local HTTP forge ([`forge_http_base`]), mirroring the prefetch's
|
|
||||||
/// former curl; Forgejo accepts `Authorization: token <admin>`.
|
|
||||||
async fn fetch_registration_token(core_token: &str) -> Result<String> {
|
|
||||||
let url = format!(
|
|
||||||
"{}/api/v1/admin/runners/registration-token",
|
|
||||||
forge_http_base()
|
|
||||||
);
|
|
||||||
let resp = reqwest::Client::new()
|
|
||||||
.get(&url)
|
|
||||||
.header("Authorization", format!("token {core_token}"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.context("GET admin/runners/registration-token")?;
|
|
||||||
let status = resp.status();
|
|
||||||
let json: Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.context("parse registration-token response")?;
|
|
||||||
if !status.is_success() {
|
|
||||||
anyhow::bail!("registration-token HTTP {status}: {json}");
|
|
||||||
}
|
|
||||||
json.get("token")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.map(str::to_owned)
|
|
||||||
.context("registration-token response missing 'token' field")
|
|
||||||
}
|
|
||||||
|
|
@ -4,7 +4,6 @@
|
||||||
//! collaborator access to for operator-curated shared content.
|
//! collaborator access to for operator-curated shared content.
|
||||||
//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`.
|
//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`.
|
||||||
|
|
||||||
mod ci_runner;
|
|
||||||
pub mod config_pr_poll;
|
pub mod config_pr_poll;
|
||||||
mod pr_merge;
|
mod pr_merge;
|
||||||
mod repos;
|
mod repos;
|
||||||
|
|
@ -267,9 +266,6 @@ pub async fn ensure_all() {
|
||||||
if let Err(e) = ensure_config_org_avatar(token).await {
|
if let Err(e) = ensure_config_org_avatar(token).await {
|
||||||
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
||||||
}
|
}
|
||||||
// Register the hive-ci Actions runner (off the container's boot path;
|
|
||||||
// no-op when CI is disabled or the runner already holds valid creds).
|
|
||||||
ci_runner::ensure_ci_runner_registered(token).await;
|
|
||||||
}
|
}
|
||||||
let Ok(containers) = crate::lifecycle::list().await else {
|
let Ok(containers) = crate::lifecycle::list().await else {
|
||||||
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
||||||
|
|
|
||||||
|
|
@ -322,23 +322,6 @@ pub async fn restart_matrix_daemon(agent_name: &str) -> Result<()> {
|
||||||
.await?)
|
.await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register the hive-ci Forgejo Actions runner: hand the freshly-minted
|
|
||||||
/// registration token to hive-priv, which writes it to the host-side
|
|
||||||
/// `/run/hive-ci/runner-token` env-file and restarts the in-container runner.
|
|
||||||
/// The forge admin token stays in hive-c0re; only the registration token
|
|
||||||
/// crosses to the (host-path) env-file the container bind-mounts read-only.
|
|
||||||
///
|
|
||||||
/// # Errors
|
|
||||||
/// Propagates the hive-priv call failure: the socket / IPC error, or the
|
|
||||||
/// root-side error when the token is rejected (empty or control characters),
|
|
||||||
/// the env-file write fails, or the runner restart exits non-zero.
|
|
||||||
pub async fn register_ci_runner(token: &str) -> Result<()> {
|
|
||||||
ok(call(&PrivRequest::RegisterCiRunner {
|
|
||||||
token: token.to_owned(),
|
|
||||||
})
|
|
||||||
.await?)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Restart a hive infrastructure container on the host (thin wrapper over
|
/// Restart a hive infrastructure container on the host (thin wrapper over
|
||||||
/// [`control_infra_container`] with `action = Restart`). hive-priv
|
/// [`control_infra_container`] with `action = Restart`). hive-priv
|
||||||
/// re-validates `container` against its root-side allowlist; callers must
|
/// re-validates `container` against its root-side allowlist; callers must
|
||||||
|
|
|
||||||
|
|
@ -489,19 +489,6 @@ pub enum PrivRequest {
|
||||||
agent_name: String,
|
agent_name: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Register the hive-ci Forgejo Actions runner: write the registration
|
|
||||||
/// token to the host-side `/run/hive-ci/runner-token` env-file (root-owned,
|
|
||||||
/// bind-mounted read-only into the container) as `TOKEN=<token>`, then
|
|
||||||
/// `systemctl --machine=hive-ci restart gitea-runner-hive.service` so the
|
|
||||||
/// runner picks up the credential. hive-c0re holds the forge admin token
|
|
||||||
/// and mints the registration token; only the registration token is written
|
|
||||||
/// here, and only to a host path — the admin token never enters the
|
|
||||||
/// container. The token is validated single-line + non-empty root-side.
|
|
||||||
RegisterCiRunner {
|
|
||||||
/// Forge runner registration token minted by hive-c0re.
|
|
||||||
token: String,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Start / stop / restart a hive infrastructure container on the host
|
/// Start / stop / restart a hive infrastructure container on the host
|
||||||
/// via `systemctl <action> container@<container>.service`. The
|
/// via `systemctl <action> container@<container>.service`. The
|
||||||
/// [`InfraContainer`] enum is the allowlist — serde rejects unknown /
|
/// [`InfraContainer`] enum is the allowlist — serde rejects unknown /
|
||||||
|
|
|
||||||
|
|
@ -331,8 +331,6 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
||||||
restart_matrix_daemon(agent_name).await
|
restart_matrix_daemon(agent_name).await
|
||||||
}
|
}
|
||||||
|
|
||||||
PrivRequest::RegisterCiRunner { ref token } => register_ci_runner(token).await,
|
|
||||||
|
|
||||||
PrivRequest::ControlInfraContainer { container, action } => {
|
PrivRequest::ControlInfraContainer { container, action } => {
|
||||||
control_infra_container(container, action).await
|
control_infra_container(container, action).await
|
||||||
}
|
}
|
||||||
|
|
@ -598,47 +596,6 @@ async fn restart_matrix_daemon(agent_name: &str) -> Result<(String, String)> {
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `RegisterCiRunner` — write the runner registration token to the host-side
|
|
||||||
/// `/run/hive-ci/runner-token` env-file, then restart the in-container runner
|
|
||||||
/// so it re-registers. The forge admin token never enters the container; only
|
|
||||||
/// the registration token c0re passes here is written, and it lands on a host
|
|
||||||
/// path bind-mounted read-only into hive-ci.
|
|
||||||
async fn register_ci_runner(token: &str) -> Result<(String, String)> {
|
|
||||||
use std::os::unix::fs::PermissionsExt as _;
|
|
||||||
// Reject anything that could corrupt the `KEY=VALUE` env-file or smuggle a
|
|
||||||
// second line — a forge registration token is an opaque single-line string.
|
|
||||||
if token.is_empty() || token.contains(['\n', '\r', '\0']) {
|
|
||||||
bail!("ci runner registration token empty or contains control characters");
|
|
||||||
}
|
|
||||||
let token_path = "/run/hive-ci/runner-token";
|
|
||||||
// In-place truncate+write of the existing inode (mirrors the prefetch's
|
|
||||||
// `echo > $FILE`), NOT a temp+rename: nspawn pins this file's inode into
|
|
||||||
// hive-ci at container start, so a rename would leave the running runner
|
|
||||||
// reading the old content. Format + perms match the tmpfiles seed and the
|
|
||||||
// prefetch: `TOKEN=<tok>`, mode 0600, root-owned.
|
|
||||||
std::fs::write(token_path, format!("TOKEN={token}\n"))
|
|
||||||
.with_context(|| format!("write {token_path}"))?;
|
|
||||||
std::fs::set_permissions(token_path, std::fs::Permissions::from_mode(0o600))
|
|
||||||
.with_context(|| format!("chmod {token_path}"))?;
|
|
||||||
// Restart the in-container runner so it reads the new token and registers.
|
|
||||||
let out = Command::new("systemctl")
|
|
||||||
.args(["--machine=hive-ci", "restart", "gitea-runner-hive.service"])
|
|
||||||
.output()
|
|
||||||
.await
|
|
||||||
.context("systemctl restart gitea-runner-hive.service in hive-ci")?;
|
|
||||||
if !out.status.success() {
|
|
||||||
bail!(
|
|
||||||
"systemctl restart gitea-runner-hive.service in hive-ci exited {}: {}",
|
|
||||||
out.status,
|
|
||||||
String::from_utf8_lossy(&out.stderr).trim()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok((
|
|
||||||
String::from_utf8_lossy(&out.stdout).into_owned(),
|
|
||||||
String::from_utf8_lossy(&out.stderr).into_owned(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `ControlInfraContainer` — start/stop/restart a hive infrastructure
|
/// `ControlInfraContainer` — start/stop/restart a hive infrastructure
|
||||||
/// container via `systemctl <verb> container@<container>.service`. The
|
/// container via `systemctl <verb> container@<container>.service`. The
|
||||||
/// [`InfraContainer`] enum is the allowlist: serde already rejected any
|
/// [`InfraContainer`] enum is the allowlist: serde already rejected any
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,147 @@ let
|
||||||
useSelfSigned = caTrust.useSelfSigned;
|
useSelfSigned = caTrust.useSelfSigned;
|
||||||
caContainerPath = caTrust.caContainerPath;
|
caContainerPath = caTrust.caContainerPath;
|
||||||
|
|
||||||
|
# hive-c0re writes its own admin token here on first forge startup.
|
||||||
|
# The token has read:admin + write:admin scopes — sufficient to call
|
||||||
|
# the runner registration-token API endpoint.
|
||||||
|
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
||||||
|
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
||||||
|
|
||||||
|
# Container state root on the host. Non-ephemeral containers keep
|
||||||
|
# their filesystem here across reboots. The prefetch service accesses
|
||||||
|
# the runner's .runner file via this path to validate credentials
|
||||||
|
# without entering the container.
|
||||||
|
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
||||||
|
|
||||||
|
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
||||||
|
#
|
||||||
|
# The core token stays on the host. The container only ever sees the
|
||||||
|
# TOKEN= env-file populated here, never the core token itself.
|
||||||
|
#
|
||||||
|
# Flow:
|
||||||
|
# 1. If .runner exists: validate the runner ID against forge.
|
||||||
|
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
||||||
|
# forge container starts and admin is provisioned — this lags
|
||||||
|
# hive-c0re.service becoming active on first boot).
|
||||||
|
# 404 → purge .runner (re-registration needed).
|
||||||
|
# 000 (forge unreachable) → keep credentials, write placeholder.
|
||||||
|
# other non-200 → purge .runner.
|
||||||
|
# Token absent after 60s → keep credentials (safe — the runner
|
||||||
|
# holds valid creds; next boot will validate properly).
|
||||||
|
# 2. If .runner absent or just purged: wait up to 60s for both
|
||||||
|
# forge-core-token to appear AND forge API to respond, then
|
||||||
|
# fetch a fresh registration token and write TOKEN=<real>.
|
||||||
|
# Exits non-zero if both timeout — systemd logs the failure;
|
||||||
|
# the placeholder from tmpfiles means the container still starts
|
||||||
|
# but the runner will error. Operator restarts the service once
|
||||||
|
# forge is healthy.
|
||||||
|
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
||||||
|
set -euo pipefail
|
||||||
|
TOKEN_FILE=/run/hive-ci/runner-token
|
||||||
|
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
||||||
|
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
||||||
|
|
||||||
|
# Validate existing .runner credentials against forge. Purge if
|
||||||
|
# the runner was deleted (404) or the file is malformed. Keep if
|
||||||
|
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
||||||
|
#
|
||||||
|
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
||||||
|
# it after provisioning the forge admin, which requires the forge
|
||||||
|
# container to start and become ready — this can lag behind
|
||||||
|
# hive-c0re.service becoming "active" on first boot). The same loop
|
||||||
|
# doubles as a wait for forge's API to become ready.
|
||||||
|
if [ -f "$RUNNER_FILE" ]; then
|
||||||
|
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
||||||
|
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
||||||
|
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
||||||
|
rm -f "$RUNNER_FILE"
|
||||||
|
else
|
||||||
|
# Wait for the core token before validating (same timeout as below).
|
||||||
|
CORE_TOKEN=""
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if [ -f "${coreTokenPath}" ]; then
|
||||||
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if [ -z "''${CORE_TOKEN:-}" ]; then
|
||||||
|
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
||||||
|
else
|
||||||
|
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-H "Authorization: token $CORE_TOKEN" \
|
||||||
|
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
||||||
|
if [ "$HTTP" = "404" ]; then
|
||||||
|
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
||||||
|
rm -f "$RUNNER_FILE"
|
||||||
|
elif [ "$HTTP" = "000" ]; then
|
||||||
|
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
||||||
|
elif [ "$HTTP" != "200" ]; then
|
||||||
|
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
||||||
|
rm -f "$RUNNER_FILE"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# .runner valid → write placeholder; gitea-actions-runner skips
|
||||||
|
# re-registration when .runner exists (TOKEN value is irrelevant).
|
||||||
|
if [ -f "$RUNNER_FILE" ]; then
|
||||||
|
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
||||||
|
chmod 600 "$TOKEN_FILE"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# First boot or stale creds purged — wait for forge-core-token, then
|
||||||
|
# fetch a fresh registration token. Single retry loop covers both:
|
||||||
|
# waiting for hive-c0re to write the token file AND for forge's API
|
||||||
|
# to become responsive (they race on first boot).
|
||||||
|
REG_TOKEN=""
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if [ ! -f "${coreTokenPath}" ]; then
|
||||||
|
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
||||||
|
sleep 1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||||
|
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
||||||
|
# distinguished from a transient forge hiccup. A bare
|
||||||
|
# `curl -sf | jq` would let a forge-core-token that is stale for
|
||||||
|
# the current forge (e.g. after a forge rebuild) 401 silently on
|
||||||
|
# every attempt for the full 60s loop and exit with a misleading
|
||||||
|
# "core token absent or forge unreachable" — masking the real cause.
|
||||||
|
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
||||||
|
# the operator/hive-c0re knows to re-mint forge-core-token.
|
||||||
|
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
||||||
|
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
||||||
|
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
||||||
|
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
||||||
|
BODY=$(printf '%s' "$RESP" | sed '$d')
|
||||||
|
case "$HTTP" in
|
||||||
|
2*)
|
||||||
|
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
||||||
|
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
||||||
|
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
||||||
|
;;
|
||||||
|
401 | 403)
|
||||||
|
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
||||||
|
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
||||||
|
chmod 600 "$TOKEN_FILE"
|
||||||
|
'';
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
||||||
|
|
@ -40,12 +181,11 @@ in
|
||||||
# on first registration and reuses them on every subsequent start).
|
# on first registration and reuses them on every subsequent start).
|
||||||
#
|
#
|
||||||
# Credential isolation: the forge admin token (`forge-core-token`)
|
# Credential isolation: the forge admin token (`forge-core-token`)
|
||||||
# never enters the hive-ci container. hive-c0re holds it and performs
|
# never enters the hive-ci container. A host-side oneshot service
|
||||||
# all forge API calls (runner validation + registration-token mint,
|
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
||||||
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
|
# the container starts and writes only the runner registration token
|
||||||
# registration token to the host env-file `/run/hive-ci/runner-token`,
|
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
||||||
# which the container bind-mounts read-only. The container never has
|
# file read-only and never has access to the wider admin token.
|
||||||
# access to the wider admin token.
|
|
||||||
#
|
#
|
||||||
# Nix builds inside the container use the shared /nix/store (standard
|
# Nix builds inside the container use the shared /nix/store (standard
|
||||||
# nixos-container behaviour) with sandbox-fallback = true, because
|
# nixos-container behaviour) with sandbox-fallback = true, because
|
||||||
|
|
@ -153,34 +293,83 @@ in
|
||||||
];
|
];
|
||||||
|
|
||||||
# Create /run/hive-ci/ on the host and seed runner-token with a
|
# Create /run/hive-ci/ on the host and seed runner-token with a
|
||||||
# placeholder. The container bind-mounts this file read-only; hive-c0re
|
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
||||||
# (via hive-priv's RegisterCiRunner) overwrites it with the real
|
# token (or a fresh placeholder) before the container starts. The
|
||||||
# registration token when it registers the runner out of band. The
|
# placeholder ensures the EnvironmentFile is always present even if
|
||||||
# placeholder keeps the runner's EnvironmentFile present from first boot,
|
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
||||||
# before c0re has registered — the runner's ExecStartPre precond (below)
|
|
||||||
# distinguishes the placeholder from a real token.
|
|
||||||
systemd.tmpfiles.rules = [
|
systemd.tmpfiles.rules = [
|
||||||
"d /run/hive-ci 0700 root root -"
|
"d /run/hive-ci 0700 root root -"
|
||||||
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
||||||
];
|
];
|
||||||
|
|
||||||
# Tell hive-c0re that CI is enabled so its startup sweep registers the
|
# Host-side oneshot: validates/refreshes runner credentials before
|
||||||
# runner. Registration moved OFF the container's boot-critical path into
|
# the container starts. The core admin token stays on the host and
|
||||||
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
|
# is never bind-mounted into the container. Runs on every boot so
|
||||||
# against the forge and, when absent/stale, mints a registration token and
|
# stale .runner credentials (runner deleted from forge) are detected
|
||||||
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
|
# and the container re-registers on the next start.
|
||||||
# runner. The forge admin token stays in hive-c0re; only the registration
|
systemd.services.hive-ci-prefetch = {
|
||||||
# token reaches the host env-file the container mounts read-only.
|
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
||||||
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
|
# Run before the container starts but after tmpfiles so the token
|
||||||
|
# file directory exists. After hive-c0re so the forge token is
|
||||||
|
# likely written (best-effort — the script handles the absent case).
|
||||||
|
after = [
|
||||||
|
"systemd-tmpfiles-setup.service"
|
||||||
|
"hive-c0re.service"
|
||||||
|
];
|
||||||
|
before = [ "container@hive-ci.service" ];
|
||||||
|
wantedBy = [ "container@hive-ci.service" ];
|
||||||
|
# partOf binds this oneshot's lifecycle to the container: when the
|
||||||
|
# container is stopped or restarted, systemd propagates that to this
|
||||||
|
# unit so it re-runs on the NEXT container start. Without this, the
|
||||||
|
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
||||||
|
# its first run — so a container restart skips it and the stale
|
||||||
|
# runner-token file (a placeholder from a boot where the forge token
|
||||||
|
# wasn't ready yet, or a registration token consumed/rotated since)
|
||||||
|
# is never refreshed. The in-container register service then fails
|
||||||
|
# with "runner registration token not found". partOf guarantees a
|
||||||
|
# fresh token is fetched before every container (re)start.
|
||||||
|
#
|
||||||
|
# Unit name: a declarative `containers.<name>` is the host unit
|
||||||
|
# `container@<name>.service` (the nspawn template), NOT
|
||||||
|
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
||||||
|
# matched no real unit, so before/wantedBy/partOf were silent
|
||||||
|
# no-ops — the partOf never bound, the oneshot stayed
|
||||||
|
# `active (exited)`, and the token was never refreshed on restart.
|
||||||
|
# Confirmed against the live `container@hive-matrix.service` unit
|
||||||
|
# during the matrix-outage incident.
|
||||||
|
partOf = [ "container@hive-ci.service" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
ExecStart = prefetchScript;
|
||||||
|
# Pin the journal identity; ExecStart is a writeShellScript whose
|
||||||
|
# store-path basename would otherwise be the journal identifier.
|
||||||
|
SyslogIdentifier = "hive-ci-prefetch";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
||||||
# in self-signed mode (see the hive-ca-trust helper). Runner registration
|
# in self-signed mode (see the hive-ca-trust helper); merged with the
|
||||||
# is no longer on the boot-critical path — hive-c0re owns it out of band —
|
# hive-ci-specific start-timeout bump below.
|
||||||
# so the container boots immediately and needs no start-timeout bump. The
|
systemd.services."container@hive-ci" = lib.mkMerge [
|
||||||
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
|
caTrust.containerOrdering
|
||||||
# boot-path forge + core-token round-trip that could trip the nspawn start
|
{
|
||||||
# timeout into a ~60s restart loop) is gone with that move.
|
# gitea-runner registration (hive-ci-prefetch, host-side)
|
||||||
systemd.services."container@hive-ci" = caTrust.containerOrdering;
|
# sits on the boot-critical path — the container's nspawn readiness
|
||||||
|
# is gated on the runner registering, a forge + core-token round
|
||||||
|
# trip that itself waits up to 60s for the core token. The default
|
||||||
|
# nspawn `TimeoutStartSec` (~60s, systemd's `DefaultTimeoutStartSec`)
|
||||||
|
# can therefore trip mid-register, especially right after a
|
||||||
|
# `.runner`-purge (every boot re-registers from scratch — see
|
||||||
|
# hive-ci-prefetch above) or during a hive-c0re restart storm that
|
||||||
|
# delays the core token. systemd then kills the half-started
|
||||||
|
# container and reschedules, producing an observed ~60s restart
|
||||||
|
# loop until the forge/token settle. Bumping the timeout well past
|
||||||
|
# the prefetch's own 60s wait gives one register attempt room to
|
||||||
|
# actually finish instead of being killed mid-flight.
|
||||||
|
serviceConfig.TimeoutStartSec = lib.mkForce "180s";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
containers.hive-ci = {
|
containers.hive-ci = {
|
||||||
autoStart = true;
|
autoStart = true;
|
||||||
|
|
@ -194,10 +383,9 @@ in
|
||||||
hostBridge = networkCfg.bridgeName;
|
hostBridge = networkCfg.bridgeName;
|
||||||
|
|
||||||
bindMounts = {
|
bindMounts = {
|
||||||
# Pre-filled by hive-c0re (via hive-priv) with the runner
|
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
||||||
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
|
# container starts. Read-only: the container reads TOKEN= from
|
||||||
# that. Read-only: the container reads TOKEN= from here; the core
|
# here; the core admin token never enters the container.
|
||||||
# admin token never enters the container.
|
|
||||||
"/run/hive-ci/runner-token" = {
|
"/run/hive-ci/runner-token" = {
|
||||||
hostPath = "/run/hive-ci/runner-token";
|
hostPath = "/run/hive-ci/runner-token";
|
||||||
isReadOnly = true;
|
isReadOnly = true;
|
||||||
|
|
@ -258,11 +446,10 @@ in
|
||||||
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
||||||
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
||||||
url = "http://${forgeCfg.domain}";
|
url = "http://${forgeCfg.domain}";
|
||||||
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
|
# EnvironmentFile providing TOKEN= — pre-filled by the
|
||||||
# (via hive-priv) with the runner registration token;
|
# host-side hive-ci-prefetch.service before the container
|
||||||
# bind-mounted read-only from /run/hive-ci/runner-token on the
|
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
||||||
# host. The runner's ExecStartPre precond gates on this being a
|
# on the host.
|
||||||
# real (non-placeholder) token or an already-registered .runner.
|
|
||||||
tokenFile = "/run/hive-ci/runner-token";
|
tokenFile = "/run/hive-ci/runner-token";
|
||||||
labels = cfg.labels;
|
labels = cfg.labels;
|
||||||
settings = {
|
settings = {
|
||||||
|
|
@ -278,9 +465,9 @@ in
|
||||||
# automatically before launching the container's init.
|
# automatically before launching the container's init.
|
||||||
|
|
||||||
# No hive-ci-register.service inside the container: all forge
|
# No hive-ci-register.service inside the container: all forge
|
||||||
# API calls (runner validation, token fetch) live in hive-c0re
|
# API calls (runner validation, token fetch) moved to the
|
||||||
# (forge/ci_runner.rs), out of band. The core admin token never
|
# host-side hive-ci-prefetch.service. The core admin token
|
||||||
# enters this container.
|
# never enters this container.
|
||||||
|
|
||||||
# git is already in the gitea-actions-runner service PATH (the
|
# git is already in the gitea-actions-runner service PATH (the
|
||||||
# nixpkgs module builds it from the package's runtime deps).
|
# nixpkgs module builds it from the package's runtime deps).
|
||||||
|
|
@ -329,28 +516,6 @@ in
|
||||||
# daemon is in-container or a shared host socket). `mkBefore` so
|
# daemon is in-container or a shared host socket). `mkBefore` so
|
||||||
# this runs ahead of any pre-steps the upstream module adds.
|
# this runs ahead of any pre-steps the upstream module adds.
|
||||||
serviceConfig.ExecStartPre = lib.mkBefore [
|
serviceConfig.ExecStartPre = lib.mkBefore [
|
||||||
# Fail fast unless the runner can actually come up. Registration
|
|
||||||
# is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
|
|
||||||
# a token and, via hive-priv, writes /run/hive-ci/runner-token +
|
|
||||||
# restarts this unit — so the runner never does the forge
|
|
||||||
# round-trip on the boot path (that was the old prefetch's
|
|
||||||
# boot-critical wait that could trip the nspawn start timeout).
|
|
||||||
# Pass iff already registered (.runner present) OR a real
|
|
||||||
# (non-placeholder) token is in place; otherwise exit 1 so the
|
|
||||||
# runner just stays down (Restart=on-failure retries) until c0re
|
|
||||||
# registers it, instead of blocking the container's start. Runs
|
|
||||||
# ahead of the nix-daemon wait below — no point waiting for the
|
|
||||||
# daemon if there are no credentials to come up with.
|
|
||||||
(pkgs.writeShellScript "hive-ci-runner-precond" ''
|
|
||||||
RUNNER=/var/lib/gitea-runner/hive/.runner
|
|
||||||
TOKEN_FILE=/run/hive-ci/runner-token
|
|
||||||
if [ -f "$RUNNER" ]; then exit 0; fi
|
|
||||||
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
|
|
||||||
exit 1
|
|
||||||
'')
|
|
||||||
(pkgs.writeShellScript "wait-nix-daemon" ''
|
(pkgs.writeShellScript "wait-nix-daemon" ''
|
||||||
# Up to ~180s; the daemon is normally up within seconds, this
|
# Up to ~180s; the daemon is normally up within seconds, this
|
||||||
# only bites in the post-restart cold window. Non-fatal shape:
|
# only bites in the post-restart cold window. Non-fatal shape:
|
||||||
|
|
@ -382,15 +547,6 @@ in
|
||||||
exit 1
|
exit 1
|
||||||
'')
|
'')
|
||||||
];
|
];
|
||||||
# Registration is out of band: hive-c0re explicitly restarts this
|
|
||||||
# unit once it writes the real token, which is the primary path. As
|
|
||||||
# a safety net, if the precond above fires first (no creds yet at
|
|
||||||
# boot) retry until c0re registers, rather than staying down — with
|
|
||||||
# no start-limit rate cap so it keeps retrying however long forge /
|
|
||||||
# the core token take to settle.
|
|
||||||
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
||||||
serviceConfig.RestartSec = 15;
|
|
||||||
startLimitIntervalSec = 0;
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue