vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

swarm-controller reads its forge token from the store, not a host file

Browse source 
The controller's forge token never entered bao: the forge minted it in its
container, hive-forge-swarm-controller-token copied it to a host file, and the
controller LoadCredential'ed that file. forgeTokenFile defaulted to that path
only where the forge ran, so a controller on any other host had no forge
access unless an operator copied the file by hand.

Now the forge's host publishes it and the controller reads it:

- swarm-bao-forge-publisher (replaces hive-forge-swarm-controller-token)
  waits for the in-container token, logs in with its own forge-publisher leaf
  and writes it to swarm/controller/swarm-controller/forge/token. It runs on
  every forge container start, the old unit's trigger.
- New principal swarm-forge-publisher: leaf minted by glue-bao-tls.nix,
  policy create/update on that one path, written by
  swarm-bao-forge-publisher-policy, CN reserved as a hive name.
- The controller policy gains read on that path. swarm-controller reads it
  at start under its own certificate with the OIDC secret's retry, and runs
  without forge access if it still cannot.
- forgeTokenFile, its LoadCredential and the forgeEnv gate are gone;
  SWARM_CONTROLLER_FORGE_URL is always set. hostSwarmControllerTokenFile is
  removed, and a tmpfiles rule deletes the old host copy on the forge host.

The in-container file stays: the mint unit's only-once guard reads it, and
it is the publisher's source.

Refs #4998
This commit is contained in:
atlas 2026-10-08 13:23:17 +02:00
commit 121c4678aa
18 changed files with 580 additions and 277 deletions
Download patch file Download diff file Expand all files Collapse all files

177
nix/host-modules/hive-forge/default.nix
View file

@ -56,6 +56,16 @@ let
# `forgeSecretPath` above.
swarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";
# Where `swarm-bao-forge-publisher` puts it: `controllerForgeTokenLeaf` in
# ../swarm-bao.nix, under the KV mount.
controllerTokenStorePath = "secret/swarm/controller/swarm-controller/forge/token";
baoCfg = config.services.hyperhive.swarm.bao;
storeRetry = import ../lib/store-retry.nix { };
havePublisherIdentity =
deployCfg.bao.forgePublisherClientCertFile != null
&& deployCfg.bao.forgePublisherClientKeyFile != null;
caTrust = import ../lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
# ROOT_URL forgejo advertises in clone links + outbound URLs.
@ -101,7 +111,18 @@ in
# `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
# restart. See `docs/networking/gateway.md::hive-forge container shape`.
imports = [ ./service.nix ];
imports = [
./service.nix
(lib.mkRemovedOptionModule
[ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
''
The forge publishes the swarm-controller's forge token to the swarm
secret store (swarm-bao-forge-publisher), and the controller reads it
there. Remove this definition; the file it named is read by nothing now
and can be deleted.
''
)
];
# What ./service.nix declares is what the forge IS from any hive's point of
# view: the names and ports it answers on, the URLs it advertises, and the
@ -226,28 +247,6 @@ in
eval error.
'';
};
hostSwarmControllerTokenFile = lib.mkOption {
type = lib.types.str;
default = "/var/lib/hyperhive-forge/swarm-controller.token";
description = ''
Host path where this forge deposits the freshly-minted forge
access token for the swarm's `swarm-controller` account (see
`systemd.services.forgejo-swarm-controller-account` inside the
forge container, and `hive-forge-swarm-controller-token` on the
host, which copies the token out).
Same role for this token as
`services.hyperhive.deploy.authelia.hostClientSecretDir` plays
for the OIDC secret: a **host**-local path (not inside any
container), read directly by `swarm-controller.nix`'s
`LoadCredential` when the controller runs on this same host.
On any other host the token has to get there somehow — copy it
out of this path with whatever secret management this
deployment already uses, the same shape `swarm.nix`'s own
`clientSecretFile` documents for the analogous cross-host case.
'';
};
};
config = lib.mkIf deployCfg.forgejo.enable {
@ -1049,53 +1048,119 @@ in
'';
};
# The reverse direction of the unit above: collect the
# swarm-controller's freshly-minted forge token OUT of the container
# onto a host path other units/hosts can read. Same "only the host can
# see both trees" reasoning, same ordering constraint — this can only
# depend on the container being up (`container@hive-forge.service`),
# not on the specific in-container oneshot that mints the token
# (containers run their own systemd instance, invisible to this one by
# unit name) — so it polls the same bounded way
# `hive-forge-oidc-secret` does while waiting on authelia above.
# The reverse direction of the unit above: publish the swarm-controller's
# freshly-minted forge token OUT of the container into the swarm secret
# store, where the controller reads it wherever it runs. On the host for the
# reason `hive-forge-oidc-secret` gives, and under the forge's own store
# identity, so nothing here asks where the store or the controller is.
#
# Unconditional, like the in-container unit it collects from: forge is
# a swarm-wide singleton, so the token always gets minted and always
# gets collected here, regardless of whether swarm-controller runs on
# this host, another host, or nowhere in this swarm at all.
systemd.services.hive-forge-swarm-controller-token = {
description = "collect the swarm-controller's forge token onto the host";
after = [ "container@hive-forge.service" ];
# It can only depend on the container being up, not on the in-container
# oneshot that mints the token (containers run their own systemd instance,
# invisible to this one by unit name), so it polls the same bounded way
# `hive-forge-oidc-secret` does. `wantedBy` the container, so every
# container start re-publishes whatever the mint unit left there.
#
# Unconditional, like the in-container unit it publishes from: forge is a
# swarm-wide singleton, so the token always gets minted and published,
# regardless of whether swarm-controller runs on this host, another host,
# or nowhere in this swarm at all. A host with no publisher leaf fails
# here, naming the options, rather than rendering no unit.
systemd.services.swarm-bao-forge-publisher = {
description = "publish the swarm-controller's forge token to the swarm secret store";
# The store's units exist only where the store runs; `Requires=` on an
# absent unit fails the job.
after = [
"container@hive-forge.service"
]
++ lib.optionals deployCfg.bao.enable [
"swarm-bao-pki.service"
"container@${baoCfg.machine}.service"
];
wants = lib.optionals deployCfg.bao.enable [ "container@${baoCfg.machine}.service" ];
requires = lib.optionals deployCfg.bao.enable [ "swarm-bao-pki.service" ];
wantedBy = [ "container@hive-forge.service" ];
serviceConfig = {
path = [
deployCfg.bao.package
pkgs.coreutils
];
# ../lib/store-retry.nix: a sealed or unreachable store is retried, and
# so is a token the forge has not minted yet.
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
serviceConfig = storeRetry.serviceConfig // {
Type = "oneshot";
RemainAfterExit = true;
SyslogIdentifier = "hive-forge-swarm-controller-token";
SyslogIdentifier = "swarm-bao-forge-publisher";
# Longer than the 120s bounded wait below. `DefaultTimeoutStartSec`
# is 90s, so without this systemd kills the unit at 90 — before it
# can emit the message naming the file it was waiting for.
TimeoutStartSec = "180s";
};
path = [ pkgs.coreutils ];
script = ''
set -euo pipefail
environment = {
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
}
// lib.optionalAttrs havePublisherIdentity {
BAO_CLIENT_CERT = deployCfg.bao.forgePublisherClientCertFile;
BAO_CLIENT_KEY = deployCfg.bao.forgePublisherClientKeyFile;
}
// lib.optionalAttrs (deployCfg.bao.serverCaFile != null) {
BAO_CACERT = deployCfg.bao.serverCaFile;
};
script =
if !havePublisherIdentity then
''
echo "the forge's host has no forge-publisher store identity, so it cannot publish the swarm-controller's forge token: set services.hyperhive.deploy.bao.forgePublisherClientCertFile and forgePublisherClientKeyFile (they default to the leaf the store mints only where services.hyperhive.deploy.bao.enable is set on this host)" >&2
exit 1
''
else
''
set -euo pipefail
src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"}
dst=${lib.escapeShellArg deployCfg.forgejo.hostSwarmControllerTokenFile}
src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"}
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "forge has not minted $src after 120s" >&2
exit 1
fi
for _ in $(seq 1 60); do
[ -s "$src" ] && break
sleep 2
done
if [ ! -s "$src" ]; then
echo "forge has not minted $src after 120s" >&2
exit 1
fi
install -D -m 0400 "$src" "$dst"
'';
err="$(mktemp)"
trap 'rm -f "$err"' EXIT
# `-token-only` answers on stdout and skips the token helper, a
# `sh` this unit's `path` does not carry.
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
echo "could not log in to swarm-bao with the forge publisher's certificate; the swarm-controller's forge token is not published." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
export BAO_TOKEN
# `value=@$src` hands bao the PATH: bao opens the file itself, so
# the token is never an argument of this process.
if ! bao kv put ${lib.escapeShellArg controllerTokenStorePath} value=@"$src" >/dev/null 2>"$err"; then
echo "swarm-bao refused the write to ${controllerTokenStorePath}." >&2
if [ -s "$err" ]; then
cat "$err" >&2
else
echo "bao failed without writing a diagnostic." >&2
fi
exit 1
fi
echo "published the swarm-controller's forge token to ${controllerTokenStorePath}"
'';
};
# Unused: nothing writes or reads this host path, and it may hold a copy of
# the controller's forge token.
systemd.tmpfiles.rules = [ "r /var/lib/hyperhive-forge/swarm-controller.token" ];
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
allowedTCPPorts = [
cfg.httpPort