mara/hyperhive
mara/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

wait for grafana's socket and read the forge's user list whole in the mint

Browse source
This commit is contained in:
atlas 2026-11-02 00:22:25 +01:00
commit 279428a66f
4 changed files with 95 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

26
nix/host-modules/hive-forge/default.nix
View file

@ -71,6 +71,12 @@ let
baoCfg = config.services.hyperhive.swarm.bao;
storeRetry = import ../lib/store-retry.nix { };
retryInJob = import ../lib/retry-in-job.nix { };
# How long the controller-account mint re-checks that the forge lists the
# account it just ensured. The mint is in the forge container's boot wait,
# whose `container@` limit is a minute.
accountRecheckSec = 15;
storeRefreshTimer = import ../lib/store-refresh-timer.nix { };
atomicWriteSecret = import ../lib/atomic-write-secret.nix { };
refreshConsumer = import ../lib/refresh-consumer.nix { };
@ -1052,14 +1058,28 @@ in
# an already-provisioned non-admin account to work around
# that (every hive from before this fix has one) — she
# promotes those by hand instead.
if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
${retryInJob}
# The list is read whole before `grep` sees it: `grep -q` leaves
# at its first match, and under `pipefail` forgejo's write to the
# closed pipe would fail the pipeline for an account that is
# listed.
account_listed() {
local list
list="$(forgejo admin user list "$@")" || return 1
grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]" <<<"$list"
}
if ! account_listed; then
forgejo admin user create \
--username ${lib.escapeShellArg swarmControllerForgeUser} \
--email ${lib.escapeShellArg "${swarmControllerForgeUser}@hyperhive.local"} \
--admin --random-password --must-change-password=false
echo "created forge account ${swarmControllerForgeUser}"
fi
if ! forgejo admin user list | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
# This unit is part of the container's boot wait, so the re-check
# stays well under its start limit (`accountRecheckSec`).
if ! retry_in_job ${toString accountRecheckSec} account_listed; then
echo "forge account ${swarmControllerForgeUser} absent after creation" >&2
exit 1
fi
@ -1099,7 +1119,7 @@ in
# The revocation below goes through the forge's admin API,
# which refuses a token of an account that is not a site
# admin; each retry would then leave one more token live.
if ! forgejo admin user list --admin | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
if ! account_listed --admin; then
echo "forge account ${swarmControllerForgeUser} is not a site admin; promote it by hand. No token minted." >&2
exit 1
fi

24
nix/host-modules/swarm-grafana.nix
View file

@ -427,6 +427,12 @@ let
privateNetwork = true;
storeRetry = import ./lib/store-retry.nix { };
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
retryInJob = import ./lib/retry-in-job.nix { };
# How long the controller-token unit waits for Grafana's socket before it
# fails into `storeRetry`'s restart. `container@` is ready when the guest's
# boot finishes, which can be before Grafana listens.
socketWaitSec = 60;
in
{
# What the service IS to every hive is `swarm.grafana` in
@ -1127,9 +1133,9 @@ in
SyslogIdentifier = "swarm-bao-grafana-controller-token";
# Grafana's answers, the new token among them, pass through a temp file.
PrivateTmp = true;
# Each call below is bounded by curl's own timeouts; a run makes up to
# about ten of them.
TimeoutStartSec = "3min";
# The socket wait (`socketWaitSec`) plus the calls below, each bounded
# by curl's own timeouts; a run makes up to about ten of them.
TimeoutStartSec = "4min";
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
@ -1173,6 +1179,18 @@ in
exit 1
}
${retryInJob}
# Any HTTP answer counts; a refused connection is curl's exit status 7.
grafana_listening() {
curl --silent --output /dev/null --connect-timeout 5 --max-time 15 \
--unix-socket ${lib.escapeShellArg socketPath} http://localhost/api/health
}
if ! retry_in_job ${toString socketWaitSec} grafana_listening; then
echo "Grafana's socket ${socketPath} did not answer within ${toString socketWaitSec} seconds." >&2
exit 1
fi
if [ ! -s ${lib.escapeShellArg adminPasswordPath} ]; then
echo "no Grafana admin password at ${adminPasswordPath} yet; swarm-bao-grafana-admin writes it." >&2
exit 1

27
nix/module-eval/forge-placement.nix
View file

@ -356,7 +356,7 @@ let
order = l: lib.length l < 2 || (at (lib.elemAt l 0) < at (lib.elemAt l 1) && order (lib.tail l));
steps = [
"if [ -s \"$name_path\" ]; then"
"forgejo admin user list --admin"
"account_listed --admin"
"bao login -method=cert"
"forgejo admin user generate-access-token"
"bao kv put"
@ -391,6 +391,31 @@ let
))
&& lib.elem "forgejo-swarm-controller-account.service" c.swarm-bridge-dns.before;
}
{
# The mint is in the forge container's boot wait (a oneshot in
# `multi-user.target`), so what it waits for inside its start stays under
# `container@`'s start limit. The account check reads the list whole: a
# `grep -q` on a pipe under `pipefail` fails an account that is listed.
name = "the forge's mint re-checks the account for a bounded time, under the container's start limit, and reads the list without a pipe";
ok =
let
u = mint allLocal;
s = u.script;
parts = lib.splitString "if ! retry_in_job " s;
budget = lib.toInt (lib.head (lib.splitString " " (lib.elemAt parts 1)));
in
lib.length parts == 2
&& lib.hasInfix "retry_in_job() {" s
&& lib.hasInfix "${toString budget} account_listed; then" s
&& budget > 0
&& budget <= 20
&& allLocal.containers.hive-forge.timeoutStartSec == "1min"
&& lib.elem "multi-user.target" u.wantedBy
&& lib.hasInfix "list=\"$(forgejo admin user list \"$@\")\"" s
&& lib.hasInfix "account_listed --admin" s
&& !(lib.hasInfix "forgejo admin user list |" s)
&& !(lib.hasInfix "forgejo admin user list --admin |" s);
}
{
# The absence arm: a forge host without the store has no leaf unless one
# is placed, and the mint says which options to set before it mints.

24
nix/module-eval/grafana.nix
View file

@ -310,6 +310,30 @@ let
&& lib.elem "swarm-bao-grafana-admin.service" unit.after
&& grafanaRemoteAuthelia.systemd.timers ? swarm-bao-grafana-controller-token;
}
{
# The container is ready when the guest's boot finishes, which can be
# before Grafana listens, so the unit waits for the socket inside its own
# start job, ahead of every call that needs it. The wait is bounded and
# sits inside the unit's own timeout, or systemd kills the job first.
name = "the controller's Grafana token waits a bounded time for Grafana's socket before its first call";
ok =
let
unit = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-controller-token;
s = unit.script;
parts = lib.splitString "if ! retry_in_job " s;
budget = lib.toInt (lib.head (lib.splitString " " (lib.elemAt parts 1)));
at = needle: lib.stringLength (lib.head (lib.splitString needle s));
in
lib.length parts == 2
&& lib.hasInfix "retry_in_job() {" s
&& lib.hasInfix "${toString budget} grafana_listening; then" s
&& lib.hasInfix "http://localhost/api/health" s
&& budget > 0
&& budget <= 120
&& unit.serviceConfig.TimeoutStartSec == "4min"
&& at "retry_in_job ${toString budget} grafana_listening" < at "no Grafana admin password at"
&& at "retry_in_job ${toString budget} grafana_listening" < at "status=\"$(grafana \"$admin\" GET";
}
{
# Same 403-not-a-miss reason as the matrix and queue arms below: the
# reader's grant covers the `services` prefix, so a path outside it is