Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
90831761e9 | ||
|
|
f4bcc59152 | ||
|
|
2941a889f0 | ||
|
|
858475549a |
6 changed files with 263 additions and 226 deletions
116
hive-c0re/src/forge/ci_runner.rs
Normal file
116
hive-c0re/src/forge/ci_runner.rs
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
//! hive-ci Forgejo Actions runner registration, driven from hive-c0re.
|
||||
//!
|
||||
//! Moves the runner-registration forge round-trip OFF the container's
|
||||
//! boot-critical path (it was a host-side `hive-ci-prefetch` oneshot that
|
||||
//! gated `container@hive-ci` start — see `nix/host-modules/hive-ci.nix`).
|
||||
//! hive-c0re holds the forge admin token; here it validates the runner's
|
||||
//! existing credentials and, when they're absent or stale, mints a fresh
|
||||
//! registration token and hands it to hive-priv, which writes it to the host
|
||||
//! env-file the container bind-mounts read-only and restarts the in-container
|
||||
//! runner. The admin token never enters the container — only the registration
|
||||
//! token does, exactly as the prefetch did.
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::forge_http_base;
|
||||
|
||||
/// Host path to the hive-ci runner's persisted credentials. The container is
|
||||
/// non-ephemeral, so `.runner` survives restarts; a present file with a sane
|
||||
/// `id` means the runner is already registered.
|
||||
const RUNNER_FILE: &str = "/var/lib/nixos-containers/hive-ci/var/lib/gitea-runner/hive/.runner";
|
||||
|
||||
/// Whether the operator enabled the CI runner. The nix module sets
|
||||
/// `HYPERHIVE_FORGE_CI_ENABLED=1` on `hive-c0re.service` when
|
||||
/// `services.hyperhive.forge.ci.enable` is on; absent means CI is off and
|
||||
/// there is no hive-ci container to register a runner for.
|
||||
fn ci_enabled() -> bool {
|
||||
std::env::var("HYPERHIVE_FORGE_CI_ENABLED")
|
||||
.is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true"))
|
||||
}
|
||||
|
||||
/// Ensure the hive-ci runner is registered against the forge. Best-effort:
|
||||
/// every failure is logged and swallowed so it never aborts the startup
|
||||
/// sweep ([`super::ensure_all`]). No-op when CI is disabled or the runner
|
||||
/// already holds valid credentials (so a healthy runner is never restarted).
|
||||
pub(super) async fn ensure_ci_runner_registered(core_token: &str) {
|
||||
if !ci_enabled() {
|
||||
return;
|
||||
}
|
||||
// Existing creds still valid on the forge → nothing to do.
|
||||
if let Some(id) = existing_runner_id()
|
||||
&& runner_valid(core_token, id).await
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Absent or stale → mint a fresh registration token and hand it to
|
||||
// hive-priv (root) to write the env-file + restart the runner.
|
||||
match fetch_registration_token(core_token).await {
|
||||
Ok(token) => {
|
||||
if let Err(e) = crate::priv_client::register_ci_runner(&token).await {
|
||||
tracing::warn!(error = ?e, "ci runner: hive-priv register_ci_runner failed");
|
||||
} else {
|
||||
tracing::info!("ci runner: registered hive-ci runner with a fresh token");
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = ?e, "ci runner: fetch registration token failed"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the runner id from the persisted `.runner` JSON, if present and sane
|
||||
/// (a `0` id is the gitea-runner "unregistered" sentinel).
|
||||
fn existing_runner_id() -> Option<u64> {
|
||||
let raw = std::fs::read_to_string(RUNNER_FILE).ok()?;
|
||||
let json: Value = serde_json::from_str(&raw).ok()?;
|
||||
let id = json.get("id")?.as_u64()?;
|
||||
(id != 0).then_some(id)
|
||||
}
|
||||
|
||||
/// `GET /admin/runners/{id}` — `true` iff the runner still exists on the forge
|
||||
/// (HTTP 200). A 404 (deleted from the admin panel) or any other status means
|
||||
/// re-registration is needed. A transport error (forge unreachable) is treated
|
||||
/// as "keep the existing creds" so a network blip never wipes a valid runner.
|
||||
async fn runner_valid(core_token: &str, id: u64) -> bool {
|
||||
let url = format!("{}/api/v1/admin/runners/{id}", forge_http_base());
|
||||
match reqwest::Client::new()
|
||||
.get(&url)
|
||||
.header("Authorization", format!("token {core_token}"))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(resp) => resp.status().is_success(),
|
||||
Err(e) => {
|
||||
tracing::warn!(error = ?e, "ci runner: validation request failed; keeping existing creds");
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /admin/runners/registration-token` — mint a fresh registration token.
|
||||
/// forgejo-api (0.11) doesn't wrap this endpoint, so this is a raw request
|
||||
/// against the local HTTP forge ([`forge_http_base`]), mirroring the prefetch's
|
||||
/// former curl; Forgejo accepts `Authorization: token <admin>`.
|
||||
async fn fetch_registration_token(core_token: &str) -> Result<String> {
|
||||
let url = format!(
|
||||
"{}/api/v1/admin/runners/registration-token",
|
||||
forge_http_base()
|
||||
);
|
||||
let resp = reqwest::Client::new()
|
||||
.get(&url)
|
||||
.header("Authorization", format!("token {core_token}"))
|
||||
.send()
|
||||
.await
|
||||
.context("GET admin/runners/registration-token")?;
|
||||
let status = resp.status();
|
||||
let json: Value = resp
|
||||
.json()
|
||||
.await
|
||||
.context("parse registration-token response")?;
|
||||
if !status.is_success() {
|
||||
anyhow::bail!("registration-token HTTP {status}: {json}");
|
||||
}
|
||||
json.get("token")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
.context("registration-token response missing 'token' field")
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
//! collaborator access to for operator-curated shared content.
|
||||
//! No-op when `hive-forge` isn't running. Full design: `docs/forge.md`.
|
||||
|
||||
mod ci_runner;
|
||||
pub mod config_pr_poll;
|
||||
mod pr_merge;
|
||||
mod repos;
|
||||
|
|
@ -266,6 +267,9 @@ pub async fn ensure_all() {
|
|||
if let Err(e) = ensure_config_org_avatar(token).await {
|
||||
tracing::warn!(error = ?e, "forge: ensure_config_org_avatar failed");
|
||||
}
|
||||
// Register the hive-ci Actions runner (off the container's boot path;
|
||||
// no-op when CI is disabled or the runner already holds valid creds).
|
||||
ci_runner::ensure_ci_runner_registered(token).await;
|
||||
}
|
||||
let Ok(containers) = crate::lifecycle::list().await else {
|
||||
tracing::warn!("forge: nixos-container list failed; skipping user sweep");
|
||||
|
|
|
|||
|
|
@ -322,6 +322,23 @@ pub async fn restart_matrix_daemon(agent_name: &str) -> Result<()> {
|
|||
.await?)
|
||||
}
|
||||
|
||||
/// Register the hive-ci Forgejo Actions runner: hand the freshly-minted
|
||||
/// registration token to hive-priv, which writes it to the host-side
|
||||
/// `/run/hive-ci/runner-token` env-file and restarts the in-container runner.
|
||||
/// The forge admin token stays in hive-c0re; only the registration token
|
||||
/// crosses to the (host-path) env-file the container bind-mounts read-only.
|
||||
///
|
||||
/// # Errors
|
||||
/// Propagates the hive-priv call failure: the socket / IPC error, or the
|
||||
/// root-side error when the token is rejected (empty or control characters),
|
||||
/// the env-file write fails, or the runner restart exits non-zero.
|
||||
pub async fn register_ci_runner(token: &str) -> Result<()> {
|
||||
ok(call(&PrivRequest::RegisterCiRunner {
|
||||
token: token.to_owned(),
|
||||
})
|
||||
.await?)
|
||||
}
|
||||
|
||||
/// Restart a hive infrastructure container on the host (thin wrapper over
|
||||
/// [`control_infra_container`] with `action = Restart`). hive-priv
|
||||
/// re-validates `container` against its root-side allowlist; callers must
|
||||
|
|
|
|||
|
|
@ -489,6 +489,19 @@ pub enum PrivRequest {
|
|||
agent_name: String,
|
||||
},
|
||||
|
||||
/// Register the hive-ci Forgejo Actions runner: write the registration
|
||||
/// token to the host-side `/run/hive-ci/runner-token` env-file (root-owned,
|
||||
/// bind-mounted read-only into the container) as `TOKEN=<token>`, then
|
||||
/// `systemctl --machine=hive-ci restart gitea-runner-hive.service` so the
|
||||
/// runner picks up the credential. hive-c0re holds the forge admin token
|
||||
/// and mints the registration token; only the registration token is written
|
||||
/// here, and only to a host path — the admin token never enters the
|
||||
/// container. The token is validated single-line + non-empty root-side.
|
||||
RegisterCiRunner {
|
||||
/// Forge runner registration token minted by hive-c0re.
|
||||
token: String,
|
||||
},
|
||||
|
||||
/// Start / stop / restart a hive infrastructure container on the host
|
||||
/// via `systemctl <action> container@<container>.service`. The
|
||||
/// [`InfraContainer`] enum is the allowlist — serde rejects unknown /
|
||||
|
|
|
|||
|
|
@ -331,6 +331,8 @@ async fn exec(req: PrivRequest, writer: &mut OwnedWriteHalf) -> Result<(String,
|
|||
restart_matrix_daemon(agent_name).await
|
||||
}
|
||||
|
||||
PrivRequest::RegisterCiRunner { ref token } => register_ci_runner(token).await,
|
||||
|
||||
PrivRequest::ControlInfraContainer { container, action } => {
|
||||
control_infra_container(container, action).await
|
||||
}
|
||||
|
|
@ -596,6 +598,47 @@ async fn restart_matrix_daemon(agent_name: &str) -> Result<(String, String)> {
|
|||
))
|
||||
}
|
||||
|
||||
/// `RegisterCiRunner` — write the runner registration token to the host-side
|
||||
/// `/run/hive-ci/runner-token` env-file, then restart the in-container runner
|
||||
/// so it re-registers. The forge admin token never enters the container; only
|
||||
/// the registration token c0re passes here is written, and it lands on a host
|
||||
/// path bind-mounted read-only into hive-ci.
|
||||
async fn register_ci_runner(token: &str) -> Result<(String, String)> {
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
// Reject anything that could corrupt the `KEY=VALUE` env-file or smuggle a
|
||||
// second line — a forge registration token is an opaque single-line string.
|
||||
if token.is_empty() || token.contains(['\n', '\r', '\0']) {
|
||||
bail!("ci runner registration token empty or contains control characters");
|
||||
}
|
||||
let token_path = "/run/hive-ci/runner-token";
|
||||
// In-place truncate+write of the existing inode (mirrors the prefetch's
|
||||
// `echo > $FILE`), NOT a temp+rename: nspawn pins this file's inode into
|
||||
// hive-ci at container start, so a rename would leave the running runner
|
||||
// reading the old content. Format + perms match the tmpfiles seed and the
|
||||
// prefetch: `TOKEN=<tok>`, mode 0600, root-owned.
|
||||
std::fs::write(token_path, format!("TOKEN={token}\n"))
|
||||
.with_context(|| format!("write {token_path}"))?;
|
||||
std::fs::set_permissions(token_path, std::fs::Permissions::from_mode(0o600))
|
||||
.with_context(|| format!("chmod {token_path}"))?;
|
||||
// Restart the in-container runner so it reads the new token and registers.
|
||||
let out = Command::new("systemctl")
|
||||
.args(["--machine=hive-ci", "restart", "gitea-runner-hive.service"])
|
||||
.output()
|
||||
.await
|
||||
.context("systemctl restart gitea-runner-hive.service in hive-ci")?;
|
||||
if !out.status.success() {
|
||||
bail!(
|
||||
"systemctl restart gitea-runner-hive.service in hive-ci exited {}: {}",
|
||||
out.status,
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
);
|
||||
}
|
||||
Ok((
|
||||
String::from_utf8_lossy(&out.stdout).into_owned(),
|
||||
String::from_utf8_lossy(&out.stderr).into_owned(),
|
||||
))
|
||||
}
|
||||
|
||||
/// `ControlInfraContainer` — start/stop/restart a hive infrastructure
|
||||
/// container via `systemctl <verb> container@<container>.service`. The
|
||||
/// [`InfraContainer`] enum is the allowlist: serde already rejected any
|
||||
|
|
|
|||
|
|
@ -24,147 +24,6 @@ let
|
|||
useSelfSigned = caTrust.useSelfSigned;
|
||||
caContainerPath = caTrust.caContainerPath;
|
||||
|
||||
# hive-c0re writes its own admin token here on first forge startup.
|
||||
# The token has read:admin + write:admin scopes — sufficient to call
|
||||
# the runner registration-token API endpoint.
|
||||
# This path is HOST-ONLY. It is never bind-mounted into hive-ci.
|
||||
coreTokenPath = "/var/lib/hyperhive/forge-core-token";
|
||||
|
||||
# Container state root on the host. Non-ephemeral containers keep
|
||||
# their filesystem here across reboots. The prefetch service accesses
|
||||
# the runner's .runner file via this path to validate credentials
|
||||
# without entering the container.
|
||||
containerRoot = "/var/lib/nixos-containers/hive-ci";
|
||||
|
||||
# Host-side oneshot. Runs before `container@hive-ci.service`.
|
||||
#
|
||||
# The core token stays on the host. The container only ever sees the
|
||||
# TOKEN= env-file populated here, never the core token itself.
|
||||
#
|
||||
# Flow:
|
||||
# 1. If .runner exists: validate the runner ID against forge.
|
||||
# Waits up to 60s for forge-core-token (hive-c0re writes it after
|
||||
# forge container starts and admin is provisioned — this lags
|
||||
# hive-c0re.service becoming active on first boot).
|
||||
# 404 → purge .runner (re-registration needed).
|
||||
# 000 (forge unreachable) → keep credentials, write placeholder.
|
||||
# other non-200 → purge .runner.
|
||||
# Token absent after 60s → keep credentials (safe — the runner
|
||||
# holds valid creds; next boot will validate properly).
|
||||
# 2. If .runner absent or just purged: wait up to 60s for both
|
||||
# forge-core-token to appear AND forge API to respond, then
|
||||
# fetch a fresh registration token and write TOKEN=<real>.
|
||||
# Exits non-zero if both timeout — systemd logs the failure;
|
||||
# the placeholder from tmpfiles means the container still starts
|
||||
# but the runner will error. Operator restarts the service once
|
||||
# forge is healthy.
|
||||
prefetchScript = pkgs.writeShellScript "hive-ci-prefetch" ''
|
||||
set -euo pipefail
|
||||
TOKEN_FILE=/run/hive-ci/runner-token
|
||||
FORGE_URL="http://127.0.0.1:${toString forgeCfg.httpPort}"
|
||||
RUNNER_FILE="${containerRoot}/var/lib/gitea-runner/hive/.runner"
|
||||
|
||||
# Validate existing .runner credentials against forge. Purge if
|
||||
# the runner was deleted (404) or the file is malformed. Keep if
|
||||
# forge is unreachable (000) — transient outage shouldn't wipe creds.
|
||||
#
|
||||
# We wait up to 60s for forge-core-token to appear (hive-c0re writes
|
||||
# it after provisioning the forge admin, which requires the forge
|
||||
# container to start and become ready — this can lag behind
|
||||
# hive-c0re.service becoming "active" on first boot). The same loop
|
||||
# doubles as a wait for forge's API to become ready.
|
||||
if [ -f "$RUNNER_FILE" ]; then
|
||||
RUNNER_ID=$(${pkgs.jq}/bin/jq -r '.id // empty' "$RUNNER_FILE" 2>/dev/null || true)
|
||||
if [ -z "''${RUNNER_ID:-}" ] || [ "$RUNNER_ID" = "0" ]; then
|
||||
echo "hive-ci-prefetch: .runner malformed (no id), purging for re-registration" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
else
|
||||
# Wait for the core token before validating (same timeout as below).
|
||||
CORE_TOKEN=""
|
||||
for i in $(seq 1 60); do
|
||||
if [ -f "${coreTokenPath}" ]; then
|
||||
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "''${CORE_TOKEN:-}" ]; then
|
||||
echo "hive-ci-prefetch: core token absent after 60s, keeping existing .runner" >&2
|
||||
else
|
||||
HTTP=$(${pkgs.curl}/bin/curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $CORE_TOKEN" \
|
||||
"$FORGE_URL/api/v1/admin/runners/$RUNNER_ID" || echo "000")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "hive-ci-prefetch: runner $RUNNER_ID gone from forge, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
elif [ "$HTTP" = "000" ]; then
|
||||
echo "hive-ci-prefetch: forge unreachable, keeping existing credentials" >&2
|
||||
elif [ "$HTTP" != "200" ]; then
|
||||
echo "hive-ci-prefetch: runner validation HTTP $HTTP, purging .runner" >&2
|
||||
rm -f "$RUNNER_FILE"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# .runner valid → write placeholder; gitea-actions-runner skips
|
||||
# re-registration when .runner exists (TOKEN value is irrelevant).
|
||||
if [ -f "$RUNNER_FILE" ]; then
|
||||
echo "TOKEN=placeholder" > "$TOKEN_FILE"
|
||||
chmod 600 "$TOKEN_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# First boot or stale creds purged — wait for forge-core-token, then
|
||||
# fetch a fresh registration token. Single retry loop covers both:
|
||||
# waiting for hive-c0re to write the token file AND for forge's API
|
||||
# to become responsive (they race on first boot).
|
||||
REG_TOKEN=""
|
||||
for i in $(seq 1 60); do
|
||||
if [ ! -f "${coreTokenPath}" ]; then
|
||||
echo "hive-ci-prefetch: waiting for core token (attempt $i/60)..." >&2
|
||||
sleep 1
|
||||
continue
|
||||
fi
|
||||
CORE_TOKEN=$(cat ${coreTokenPath})
|
||||
# Capture the HTTP status so a stale/invalid core token (401/403) is
|
||||
# distinguished from a transient forge hiccup. A bare
|
||||
# `curl -sf | jq` would let a forge-core-token that is stale for
|
||||
# the current forge (e.g. after a forge rebuild) 401 silently on
|
||||
# every attempt for the full 60s loop and exit with a misleading
|
||||
# "core token absent or forge unreachable" — masking the real cause.
|
||||
# Fail fast + loudly on 401/403 so the failure mode is legible and
|
||||
# the operator/hive-c0re knows to re-mint forge-core-token.
|
||||
RESP=$(${pkgs.curl}/bin/curl -s -w $'\n%{http_code}' \
|
||||
"$FORGE_URL/api/v1/admin/runners/registration-token" \
|
||||
-H "Authorization: token $CORE_TOKEN" || printf '\n000')
|
||||
HTTP=$(printf '%s' "$RESP" | tail -n1)
|
||||
BODY=$(printf '%s' "$RESP" | sed '$d')
|
||||
case "$HTTP" in
|
||||
2*)
|
||||
REG_TOKEN=$(printf '%s' "$BODY" | ${pkgs.jq}/bin/jq -r .token)
|
||||
if [ -n "''${REG_TOKEN:-}" ] && [ "$REG_TOKEN" != "null" ]; then break; fi
|
||||
echo "hive-ci-prefetch: 2xx but no token in response (attempt $i/60), retrying..." >&2
|
||||
;;
|
||||
401 | 403)
|
||||
echo "hive-ci-prefetch: forge rejected forge-core-token (HTTP $HTTP) — it is stale/invalid for the current forge. hive-c0re must re-mint it (delete /var/lib/hyperhive/forge-core-token, or rely on the validate-or-remint path). Failing fast instead of looping 60s." >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "hive-ci-prefetch: registration-token fetch HTTP $HTTP (attempt $i/60), retrying..." >&2
|
||||
;;
|
||||
esac
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "''${REG_TOKEN:-}" ] || [ "$REG_TOKEN" = "null" ]; then
|
||||
echo "hive-ci-prefetch: failed to fetch runner registration token (core token absent or forge unreachable after 60s)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "TOKEN=$REG_TOKEN" > "$TOKEN_FILE"
|
||||
chmod 600 "$TOKEN_FILE"
|
||||
'';
|
||||
in
|
||||
{
|
||||
# Forgejo Actions runner in a `hive-ci` nixos-container.
|
||||
|
|
@ -181,11 +40,12 @@ in
|
|||
# on first registration and reuses them on every subsequent start).
|
||||
#
|
||||
# Credential isolation: the forge admin token (`forge-core-token`)
|
||||
# never enters the hive-ci container. A host-side oneshot service
|
||||
# (`hive-ci-prefetch.service`) performs all forge API calls before
|
||||
# the container starts and writes only the runner registration token
|
||||
# to `/run/hive-ci/runner-token`. The container bind-mounts this
|
||||
# file read-only and never has access to the wider admin token.
|
||||
# never enters the hive-ci container. hive-c0re holds it and performs
|
||||
# all forge API calls (runner validation + registration-token mint,
|
||||
# in `forge/ci_runner.rs`); via hive-priv it writes only the runner
|
||||
# registration token to the host env-file `/run/hive-ci/runner-token`,
|
||||
# which the container bind-mounts read-only. The container never has
|
||||
# access to the wider admin token.
|
||||
#
|
||||
# Nix builds inside the container use the shared /nix/store (standard
|
||||
# nixos-container behaviour) with sandbox-fallback = true, because
|
||||
|
|
@ -293,83 +153,34 @@ in
|
|||
];
|
||||
|
||||
# Create /run/hive-ci/ on the host and seed runner-token with a
|
||||
# placeholder. hive-ci-prefetch.service overwrites it with the real
|
||||
# token (or a fresh placeholder) before the container starts. The
|
||||
# placeholder ensures the EnvironmentFile is always present even if
|
||||
# the prefetch service hasn't run yet (e.g. tmpfiles-setup timing).
|
||||
# placeholder. The container bind-mounts this file read-only; hive-c0re
|
||||
# (via hive-priv's RegisterCiRunner) overwrites it with the real
|
||||
# registration token when it registers the runner out of band. The
|
||||
# placeholder keeps the runner's EnvironmentFile present from first boot,
|
||||
# before c0re has registered — the runner's ExecStartPre precond (below)
|
||||
# distinguishes the placeholder from a real token.
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /run/hive-ci 0700 root root -"
|
||||
"f /run/hive-ci/runner-token 0600 root root - TOKEN=placeholder"
|
||||
];
|
||||
|
||||
# Host-side oneshot: validates/refreshes runner credentials before
|
||||
# the container starts. The core admin token stays on the host and
|
||||
# is never bind-mounted into the container. Runs on every boot so
|
||||
# stale .runner credentials (runner deleted from forge) are detected
|
||||
# and the container re-registers on the next start.
|
||||
systemd.services.hive-ci-prefetch = {
|
||||
description = "Pre-fetch hive-ci runner registration token (host-side)";
|
||||
# Run before the container starts but after tmpfiles so the token
|
||||
# file directory exists. After hive-c0re so the forge token is
|
||||
# likely written (best-effort — the script handles the absent case).
|
||||
after = [
|
||||
"systemd-tmpfiles-setup.service"
|
||||
"hive-c0re.service"
|
||||
];
|
||||
before = [ "container@hive-ci.service" ];
|
||||
wantedBy = [ "container@hive-ci.service" ];
|
||||
# partOf binds this oneshot's lifecycle to the container: when the
|
||||
# container is stopped or restarted, systemd propagates that to this
|
||||
# unit so it re-runs on the NEXT container start. Without this, the
|
||||
# RemainAfterExit=true oneshot stays "active (exited)" forever after
|
||||
# its first run — so a container restart skips it and the stale
|
||||
# runner-token file (a placeholder from a boot where the forge token
|
||||
# wasn't ready yet, or a registration token consumed/rotated since)
|
||||
# is never refreshed. The in-container register service then fails
|
||||
# with "runner registration token not found". partOf guarantees a
|
||||
# fresh token is fetched before every container (re)start.
|
||||
#
|
||||
# Unit name: a declarative `containers.<name>` is the host unit
|
||||
# `container@<name>.service` (the nspawn template), NOT
|
||||
# `nixos-container@…`. The earlier `nixos-container@hive-ci.service`
|
||||
# matched no real unit, so before/wantedBy/partOf were silent
|
||||
# no-ops — the partOf never bound, the oneshot stayed
|
||||
# `active (exited)`, and the token was never refreshed on restart.
|
||||
# Confirmed against the live `container@hive-matrix.service` unit
|
||||
# during the matrix-outage incident.
|
||||
partOf = [ "container@hive-ci.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = prefetchScript;
|
||||
# Pin the journal identity; ExecStart is a writeShellScript whose
|
||||
# store-path basename would otherwise be the journal identifier.
|
||||
SyslogIdentifier = "hive-ci-prefetch";
|
||||
};
|
||||
};
|
||||
# Tell hive-c0re that CI is enabled so its startup sweep registers the
|
||||
# runner. Registration moved OFF the container's boot-critical path into
|
||||
# hive-c0re (`forge/ci_runner.rs`): it validates the persisted `.runner`
|
||||
# against the forge and, when absent/stale, mints a registration token and
|
||||
# hands it to hive-priv to write `/run/hive-ci/runner-token` + restart the
|
||||
# runner. The forge admin token stays in hive-c0re; only the registration
|
||||
# token reaches the host env-file the container mounts read-only.
|
||||
systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_CI_ENABLED = "1";
|
||||
|
||||
# `caTrust.containerOrdering` orders this unit after `hive-tls-ca.service`
|
||||
# in self-signed mode (see the hive-ca-trust helper); merged with the
|
||||
# hive-ci-specific start-timeout bump below.
|
||||
systemd.services."container@hive-ci" = lib.mkMerge [
|
||||
caTrust.containerOrdering
|
||||
{
|
||||
# gitea-runner registration (hive-ci-prefetch, host-side)
|
||||
# sits on the boot-critical path — the container's nspawn readiness
|
||||
# is gated on the runner registering, a forge + core-token round
|
||||
# trip that itself waits up to 60s for the core token. The default
|
||||
# nspawn `TimeoutStartSec` (~60s, systemd's `DefaultTimeoutStartSec`)
|
||||
# can therefore trip mid-register, especially right after a
|
||||
# `.runner`-purge (every boot re-registers from scratch — see
|
||||
# hive-ci-prefetch above) or during a hive-c0re restart storm that
|
||||
# delays the core token. systemd then kills the half-started
|
||||
# container and reschedules, producing an observed ~60s restart
|
||||
# loop until the forge/token settle. Bumping the timeout well past
|
||||
# the prefetch's own 60s wait gives one register attempt room to
|
||||
# actually finish instead of being killed mid-flight.
|
||||
serviceConfig.TimeoutStartSec = lib.mkForce "180s";
|
||||
}
|
||||
];
|
||||
# in self-signed mode (see the hive-ca-trust helper). Runner registration
|
||||
# is no longer on the boot-critical path — hive-c0re owns it out of band —
|
||||
# so the container boots immediately and needs no start-timeout bump. The
|
||||
# former `TimeoutStartSec = mkForce "180s"` band-aid (which papered over a
|
||||
# boot-path forge + core-token round-trip that could trip the nspawn start
|
||||
# timeout into a ~60s restart loop) is gone with that move.
|
||||
systemd.services."container@hive-ci" = caTrust.containerOrdering;
|
||||
|
||||
containers.hive-ci = {
|
||||
autoStart = true;
|
||||
|
|
@ -383,9 +194,10 @@ in
|
|||
hostBridge = networkCfg.bridgeName;
|
||||
|
||||
bindMounts = {
|
||||
# Pre-filled by hive-ci-prefetch.service (host-side) before the
|
||||
# container starts. Read-only: the container reads TOKEN= from
|
||||
# here; the core admin token never enters the container.
|
||||
# Pre-filled by hive-c0re (via hive-priv) with the runner
|
||||
# registration token; tmpfiles seeds a `TOKEN=placeholder` before
|
||||
# that. Read-only: the container reads TOKEN= from here; the core
|
||||
# admin token never enters the container.
|
||||
"/run/hive-ci/runner-token" = {
|
||||
hostPath = "/run/hive-ci/runner-token";
|
||||
isReadOnly = true;
|
||||
|
|
@ -446,10 +258,11 @@ in
|
|||
# the gateway vhost `forgeCfg.domain` proxies to the forge
|
||||
# on HTTP:80 (addSSL=true, no HTTP→HTTPS redirect).
|
||||
url = "http://${forgeCfg.domain}";
|
||||
# EnvironmentFile providing TOKEN= — pre-filled by the
|
||||
# host-side hive-ci-prefetch.service before the container
|
||||
# starts; bind-mounted read-only from /run/hive-ci/runner-token
|
||||
# on the host.
|
||||
# EnvironmentFile providing TOKEN= — pre-filled by hive-c0re
|
||||
# (via hive-priv) with the runner registration token;
|
||||
# bind-mounted read-only from /run/hive-ci/runner-token on the
|
||||
# host. The runner's ExecStartPre precond gates on this being a
|
||||
# real (non-placeholder) token or an already-registered .runner.
|
||||
tokenFile = "/run/hive-ci/runner-token";
|
||||
labels = cfg.labels;
|
||||
settings = {
|
||||
|
|
@ -465,9 +278,9 @@ in
|
|||
# automatically before launching the container's init.
|
||||
|
||||
# No hive-ci-register.service inside the container: all forge
|
||||
# API calls (runner validation, token fetch) moved to the
|
||||
# host-side hive-ci-prefetch.service. The core admin token
|
||||
# never enters this container.
|
||||
# API calls (runner validation, token fetch) live in hive-c0re
|
||||
# (forge/ci_runner.rs), out of band. The core admin token never
|
||||
# enters this container.
|
||||
|
||||
# git is already in the gitea-actions-runner service PATH (the
|
||||
# nixpkgs module builds it from the package's runtime deps).
|
||||
|
|
@ -516,6 +329,28 @@ in
|
|||
# daemon is in-container or a shared host socket). `mkBefore` so
|
||||
# this runs ahead of any pre-steps the upstream module adds.
|
||||
serviceConfig.ExecStartPre = lib.mkBefore [
|
||||
# Fail fast unless the runner can actually come up. Registration
|
||||
# is done OUT OF BAND by hive-c0re (forge/ci_runner.rs): it mints
|
||||
# a token and, via hive-priv, writes /run/hive-ci/runner-token +
|
||||
# restarts this unit — so the runner never does the forge
|
||||
# round-trip on the boot path (that was the old prefetch's
|
||||
# boot-critical wait that could trip the nspawn start timeout).
|
||||
# Pass iff already registered (.runner present) OR a real
|
||||
# (non-placeholder) token is in place; otherwise exit 1 so the
|
||||
# runner just stays down (Restart=on-failure retries) until c0re
|
||||
# registers it, instead of blocking the container's start. Runs
|
||||
# ahead of the nix-daemon wait below — no point waiting for the
|
||||
# daemon if there are no credentials to come up with.
|
||||
(pkgs.writeShellScript "hive-ci-runner-precond" ''
|
||||
RUNNER=/var/lib/gitea-runner/hive/.runner
|
||||
TOKEN_FILE=/run/hive-ci/runner-token
|
||||
if [ -f "$RUNNER" ]; then exit 0; fi
|
||||
if [ -s "$TOKEN_FILE" ] && ! ${pkgs.gnugrep}/bin/grep -q '^TOKEN=placeholder$' "$TOKEN_FILE"; then
|
||||
exit 0
|
||||
fi
|
||||
echo "hive-ci runner: not registered and no real token yet; waiting for hive-c0re to register" >&2
|
||||
exit 1
|
||||
'')
|
||||
(pkgs.writeShellScript "wait-nix-daemon" ''
|
||||
# Up to ~180s; the daemon is normally up within seconds, this
|
||||
# only bites in the post-restart cold window. Non-fatal shape:
|
||||
|
|
@ -547,6 +382,15 @@ in
|
|||
exit 1
|
||||
'')
|
||||
];
|
||||
# Registration is out of band: hive-c0re explicitly restarts this
|
||||
# unit once it writes the real token, which is the primary path. As
|
||||
# a safety net, if the precond above fires first (no creds yet at
|
||||
# boot) retry until c0re registers, rather than staying down — with
|
||||
# no start-limit rate cap so it keeps retrying however long forge /
|
||||
# the core token take to settle.
|
||||
serviceConfig.Restart = lib.mkForce "on-failure";
|
||||
serviceConfig.RestartSec = 15;
|
||||
startLimitIntervalSec = 0;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue