Watch
0
0
Fork
You've already forked hyperhive
0

hive-forge: the controller-token mint writes straight to the store, and the on-disk token goes

`forgejo-swarm-controller-account`, in the forge's container, now puts
the swarm-controller's forge token into the swarm secret store itself,
under the forge publisher's leaf. The container already receives that
leaf as nspawn system credentials for its journal forwarder; the mint
loads the same three. The token reaches bao on stdin (`value=-`) and
the forge's API through a header file, never as an argument, and never
touches disk.

The mint's guard is a marker holding the published token's name, at
/var/lib/forgejo/swarm-controller-token-name, written last: after the
put, and after every other `swarm-controller-boot-*` token of the
account is revoked through forgejo's admin API, authenticated as the
new token. So a put that failed, or a marker lost while the store kept
an older token, leaves no live token behind once a mint completes. The
mint logs in to the store and checks that the account is a site admin
before it mints, so a store that is down or an account the admin API
would refuse mints nothing on each retry. Failures retry under
store-retry; forgejo does not wait for the mint. No bao read and no
grant change: the forge publisher keeps create/update on the token.

Removed: the host's `swarm-bao-forge-publisher` service and path unit,
which read the token file through the container's root, and their
entries in the store and policy ordering glue. The forge publisher's
policy unit stays; `swarm-bao-forge-oidc` still uses the leaf.

On deploy, an existing forge host has no marker, so the mint runs once:
it mints a new token, publishes it, revokes the old one, and deletes
the old /var/lib/forgejo/swarm-controller-token. The controller's next
forge call gets a 401, re-reads the store and carries on.
This commit is contained in:
atlas 2026-10-16 14:39:19 +02:00 • committed by mara
commit 141be16c6d
14 changed files with 294 additions and 261 deletions

View file

@ -55,15 +55,16 @@ let
# invite the two to drift rather than prevent it.
swarmControllerForgeUser = "swarm-controller";
# Where the minted token lands inside the forge container — under
# forgejo's own state dir, because the file is the mint's record that it
# has minted and has to survive a reboot.
swarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";
# The mint's record that it has minted and published the controller's
# token: the token's name, not its value. Inside the forge container, under
# forgejo's own state dir, so it survives a reboot.
swarmControllerTokenNamePath = "/var/lib/forgejo/swarm-controller-token-name";
# The same file from the host, which is where its publisher reads and watches it.
swarmControllerTokenHostPath = "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}";
# Where the minted token itself used to be written. The mint removes it once
# it has published a token.
oldSwarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";
# Where `swarm-bao-forge-publisher` puts it: `controllerForgeTokenLeaf` in
# Where the mint puts the token: `controllerForgeTokenLeaf` in
# ../swarm-bao.nix, under the KV mount.
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/forge/token";
@ -93,8 +94,9 @@ let
forwarderTelemetryPort = 8898;
# The forge host's store identity, handed into the container as system
# credentials (credential id → host source) for the forwarder's fetch unit.
# The host units below keep reading the same files directly.
# credentials (credential id → host source) for the forwarder's fetch unit
# and the controller-token mint. `swarm-bao-forge-oidc` on the host keeps
# reading the same files directly.
storeFiles = lib.filterAttrs (_: source: source != null) {
forge-bao-cert = deployCfg.bao.forgePublisherClientCertFile;
forge-bao-key = deployCfg.bao.forgePublisherClientKeyFile;
@ -104,6 +106,8 @@ let
# one word.
oneWord = path: builtins.match ".*[[:space:]].*" path == null;
forwarderCredential = id: "/run/credentials/${forwarderUnit}.service/${id}";
mintUnit = "forgejo-swarm-controller-account";
mintCredential = id: "/run/credentials/${mintUnit}.service/${id}";
# ROOT_URL forgejo advertises in clone links + outbound URLs.
# `cfg.domain` doubles as both the forgejo `DOMAIN` setting AND the
@ -165,9 +169,9 @@ in
[ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
''
The forge publishes the swarm-controller's forge token to the swarm
secret store (swarm-bao-forge-publisher), and the controller reads it
there. Remove this definition; the file it named is read by nothing now
and can be deleted.
secret store (forgejo-swarm-controller-account), and the controller
reads it there. Remove this definition; the file it named is read by
nothing now and can be deleted.
''
)
];
@ -540,7 +544,7 @@ in
# same reason the trust bundle below names them: an unordered
# resolver write is a race that only shows up on a cold boot. The
# forwarder and its secret fetch dial the swarm collector, authelia
# and the store by name.
# and the store by name, and the controller-token mint the store.
(import ./../../container-modules/swarm-container-resolver.nix {
inherit (networkCfg) bridgeIp;
dnsConsumers = [
@ -548,6 +552,7 @@ in
"forgejo-sso-source.service"
"opentelemetry-collector.service"
"${forwarderUnit}.service"
"${mintUnit}.service"
];
})
@ -980,34 +985,56 @@ in
# swarm-controller — the design requirement was explicit that
# this must work even when forge and swarm-controller don't
# share a host.
systemd.services.forgejo-swarm-controller-account = {
#
# The token goes straight to the swarm secret store, under the forge
# host's store identity, and never onto disk. Forgejo does not wait
# for this unit.
systemd.services.${mintUnit} = {
description = "provision the swarm-controller's forge account + access token";
after = [ "forgejo.service" ];
requires = [ "forgejo.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
# ../lib/store-retry.nix: a store that is sealed or unreachable is
# retried, and so is a forge that is not answering yet.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
User = "forgejo";
Group = "forgejo";
SyslogIdentifier = "forgejo-swarm-controller-account";
SyslogIdentifier = mintUnit;
LoadCredential = lib.attrNames storeFiles;
# The forge's answers pass through a temp file.
PrivateTmp = true;
# Each forge call is bounded by curl's own timeouts, and a sealed
# store answers on the port and never answers the write.
TimeoutStartSec = "3min";
};
# `FORGEJO_CUSTOM`, same reason as `forgejo-sso-source` above:
# every `forgejo admin` invocation needs it to find the app.ini
# the upstream module wrote, not just the auth-source verb.
environment = {
FORGEJO_CUSTOM = "/var/lib/forgejo/custom";
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs havePublisherIdentity {
BAO_CLIENT_CERT = mintCredential "forge-bao-cert";
BAO_CLIENT_KEY = mintCredential "forge-bao-key";
}
// lib.optionalAttrs (storeFiles ? forge-bao-server-ca) {
BAO_CACERT = mintCredential "forge-bao-server-ca";
};
path = [
deployCfg.forgejo.package
deployCfg.bao.package
pkgs.coreutils
pkgs.curl
pkgs.gnugrep
pkgs.jq
];
script = ''
set -euo pipefail
${atomicWriteSecret}
# Idempotent by query, same reasoning as the SSO unit: assert
# the effect, not the command's exit code. `--admin`: the
# controller creates repos in the agents org on the swarm's
@ -1032,50 +1059,133 @@ in
exit 1
fi
token_path=${lib.escapeShellArg swarmControllerTokenPath}
# Token minting is idempotent by the token FILE's existence,
# not a separate stamp: the file is both the record of "already
# done" and the thing that has to survive, so a state wipe that
# deletes it correctly triggers a re-mint instead of being
# silently masked by a stamp that outlived what it claims exists.
if [ ! -s "$token_path" ]; then
# `write:admin` added for `swarm-controller::forge::Client::
# ensure_agent_user` (the `CreateForgeUser` jobq node) —
# Forgejo's `admin_create_user` HTTP endpoint refused every
# call with "token does not have at least one of required
# scope(s): [write:admin]" without it, an already-deployed
# swarm hitting this the moment that node shipped. A host
# whose token was minted before this scope was added won't
# pick it up automatically — the `[ ! -s "$token_path" ]`
# guard above only mints when the file is absent, by design
# (see its own comment) — so an existing deployment needs
# its token file deleted to force a re-mint with the new
# scope.
#
# Token name carries a timestamp suffix rather than the
# fixed `swarm-controller-boot` this used to be — a real
# incident, not a hypothetical: deleting the token *file*
# (the remediation above) doesn't delete the token Forgejo
# already has under that name, so re-running this unit hit
# "Command error: access token name has been used already"
# and failed outright. Same fix `hive-c0re::forge::users::
# mint_token` already uses for its own agent tokens, for
# the identical reason — a monotonic suffix means a re-mint
# can never collide with whatever this account already has.
out=$(forgejo admin user generate-access-token \
--username ${lib.escapeShellArg swarmControllerForgeUser} \
--token-name "swarm-controller-boot-$(date +%s)" \
--scopes "write:repository,write:organization,write:issue,read:user,write:admin")
token=$(printf '%s' "$out" | grep -oE '[0-9a-f]{32,}' | head -n1)
if [ -z "$token" ]; then
echo "no token-shaped word in forgejo's generate-access-token output" >&2
exit 1
fi
atomic_write_secret 0600 "" "$token_path" "$token"
echo "minted forge access token for ${swarmControllerForgeUser}"
# Minted once. The marker holds the published token's name and is
# written last, so a marker present means the store has a token
# and every earlier one is revoked. Deleting it forces a re-mint,
# the step for a store that lost the token or for new scopes.
name_path=${lib.escapeShellArg swarmControllerTokenNamePath}
if [ -s "$name_path" ]; then
echo "the forge token $(< "$name_path") for ${swarmControllerForgeUser} is already in the swarm secret store"
exit 0
fi
'';
''
+ (
if !havePublisherIdentity then
''
echo "the forge's host has no forge-publisher store identity, so the swarm-controller's forge token cannot be published: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
exit 1
''
else
''
err="$(mktemp)"
body="$(mktemp)"
trap 'rm -f "$err" "$body"' EXIT
bao_failed() {
echo "$1" >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
}
# The revocation below goes through the forge's admin API,
# which refuses a token of an account that is not a site
# admin; each retry would then leave one more token live.
if ! forgejo admin user list --admin | grep -qE "[[:space:]]${swarmControllerForgeUser}[[:space:]]"; then
echo "forge account ${swarmControllerForgeUser} is not a site admin; promote it by hand. No token minted." >&2
exit 1
fi
# Before the mint, so a store that is down fails the attempt
# before forgejo holds a token nothing can read.
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
bao_failed "could not log in to swarm-bao with the forge publisher's certificate; no forge token minted for ${swarmControllerForgeUser}."
fi
export BAO_TOKEN
# `write:admin`: the controller's `CreateForgeUser` jobq node
# (`swarm-controller::forge::Client::ensure_agent_user`) and
# the revocation below call the admin API.
#
# Timestamped: forgejo refuses a name the account already has
# a token under, and the earlier tokens are still there until
# the revocation below. `hive-c0re::forge::users::mint_token`
# names its tokens the same way.
name="swarm-controller-boot-$(date +%s)"
out=$(forgejo admin user generate-access-token \
--username ${lib.escapeShellArg swarmControllerForgeUser} \
--token-name "$name" \
--scopes "write:repository,write:organization,write:issue,read:user,write:admin")
token=$(printf '%s' "$out" | grep -oE '[0-9a-f]{32,}' | head -n1)
if [ -z "$token" ]; then
echo "no token-shaped word in forgejo's generate-access-token output" >&2
exit 1
fi
# On stdin: the token is never an argument of any process.
if ! printf '%s' "$token" | bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=- >/dev/null 2>"$err"; then
bao_failed "swarm-bao refused the write to ${controllerTokenStorePath}; the next attempt mints another token and revokes this one."
fi
echo "published the forge token $name for ${swarmControllerForgeUser} to ${controllerTokenStorePath}"
# One call to the forge's API as the new token. Prints the
# HTTP status; the body lands in "$body". The token reaches
# curl as a header file, never as an argument.
forge() {
local method="$1" path="$2"
curl --silent --show-error --connect-timeout 5 --max-time 15 \
-H @<(printf 'Authorization: token %s\n' "$token") \
-X "$method" -o "$body" -w '%{http_code}' \
"http://127.0.0.1:${toString cfg.httpPort}/api/v1$path"
}
# Every other token this unit minted: one whose write to the
# store failed, and the one the store held before the marker
# was lost. The bare `swarm-controller-boot` is the name the
# unit minted under before the timestamp suffix. A failure
# here leaves the marker unwritten, so the retry mints again
# and repeats the revocation.
stale=()
page=1
while :; do
status="$(forge GET "/admin/users/${swarmControllerForgeUser}/tokens?limit=50&page=$page")"
if [ "$status" != 200 ]; then
echo "the forge answered $status to listing ${swarmControllerForgeUser}'s tokens:" >&2
cat "$body" >&2
exit 1
fi
[ "$(jq length "$body")" -gt 0 ] || break
while IFS= read -r n; do
case "$n" in
"$name") ;;
swarm-controller-boot | swarm-controller-boot-*) stale+=("$n") ;;
esac
done < <(jq -r '.[].name' "$body")
page=$((page + 1))
done
for n in "''${stale[@]}"; do
status="$(forge DELETE "/admin/users/${swarmControllerForgeUser}/tokens/$n")"
case "$status" in
204 | 404) ;;
*)
echo "the forge answered $status to revoking ${swarmControllerForgeUser}'s token $n:" >&2
cat "$body" >&2
exit 1
;;
esac
done
echo "revoked ''${#stale[@]} earlier forge token(s) of ${swarmControllerForgeUser}"
rm -f ${lib.escapeShellArg oldSwarmControllerTokenPath}
printf '%s\n' "$name" >"$name_path.tmp"
mv -f "$name_path.tmp" "$name_path"
''
);
};
};
};
@ -1089,9 +1199,9 @@ in
# read both fail loudly, and `storeRetry` retries until the publisher on
# authelia's host has written the secret.
#
# On the host, where that leaf is: the container holds no store identity.
# A host with no publisher leaf fails here, naming the options, rather than
# rendering no unit.
# On the host, which reads that leaf from its own files. A host with no
# publisher leaf fails here, naming the options, rather than rendering no
# unit.
systemd.services.swarm-bao-forge-oidc = {
description = "fetch the forge's OIDC client secret from the swarm secret store";
# Ordered after the store where it runs here
@ -1185,122 +1295,6 @@ in
systemd.timers.swarm-bao-forge-oidc = storeRefreshTimer "re-read the forge's OIDC client secret from the swarm secret store";
# The reverse direction of the unit above: publish the swarm-controller's
# freshly-minted forge token OUT of the container into the swarm secret
# store, where the controller reads it wherever it runs. On the host and
# under the forge's own store identity for the reason the unit above gives,
# so nothing here asks where the store or the controller is.
#
# It runs at boot and on a switch that adds or changes it
# (`multi-user.target`), each time the container starts, and each time the
# mint writes the file
# (`systemd.paths.swarm-bao-forge-publisher` below). The mint is a oneshot in
# the container's own systemd, invisible to this one by unit name, so a
# missing file is not an error: the path unit runs this again when the mint
# writes it, the shape of `swarm-secret-publish`.
#
# ⚠️ No `RemainAfterExit`. A oneshot that stays `active (exited)` turns
# every later start into a no-op, so neither trigger would run it again.
#
# Unconditional, like the in-container unit it publishes from: forge is a
# swarm-wide singleton, so the token always gets minted and published,
# regardless of whether swarm-controller runs on this host, another host,
# or nowhere in this swarm at all. A host with no publisher leaf fails
# here, naming the options, rather than rendering no unit.
systemd.services.swarm-bao-forge-publisher = {
description = "publish the swarm-controller's forge token to the swarm secret store";
# Also ordered after the store where it runs here:
# ../glue-bao-readers-store-order.nix.
after = [
"container@hive-forge.service"
];
wantedBy = [
"multi-user.target"
"container@hive-forge.service"
];
path = [
deployCfg.bao.package
pkgs.coreutils
];
# ../lib/store-retry.nix: a sealed or unreachable store is retried.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
SyslogIdentifier = "swarm-bao-forge-publisher";
# Bounded here so the number a switch waits on is in this file. A sealed
# store answers on the port and never answers the write.
TimeoutStartSec = 60;
};
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs havePublisherIdentity {
BAO_CLIENT_CERT = deployCfg.bao.forgePublisherClientCertFile;
BAO_CLIENT_KEY = deployCfg.bao.forgePublisherClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !havePublisherIdentity then
''
echo "the forge's host has no forge-publisher store identity, so it cannot publish the swarm-controller's forge token: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
exit 1
''
else
''
set -euo pipefail
src=${lib.escapeShellArg swarmControllerTokenHostPath}
if [ ! -s "$src" ]; then
echo "the forge has not minted $src yet; the path unit publishes it when it does" >&2
exit 0
fi
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the forge publisher's certificate; the swarm-controller's forge token is not published." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
# `value=@$src` hands bao the PATH: bao opens the file itself, so
# the token is never an argument of this process.
if ! bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=@"$src" >/dev/null 2>"$err"; then
echo "swarm-bao refused the write to ${controllerTokenStorePath}." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
echo "published the swarm-controller's forge token to ${controllerTokenStorePath}"
'';
};
# Re-publish when the mint unit writes a token, which it can do without the
# container restarting. The file is the event: the mint runs in the
# container's own systemd, which this one cannot subscribe to.
systemd.paths.swarm-bao-forge-publisher = {
description = "watch the swarm-controller's forge token for a publish";
wantedBy = [ "multi-user.target" ];
pathConfig = {
PathChanged = swarmControllerTokenHostPath;
Unit = "swarm-bao-forge-publisher.service";
};
};
systemd.tmpfiles.rules = [
# The OIDC secret's directory, which the container binds at start.
"d ${oidcSecretDir} 0700 root root -"