vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Activity

forge: hand the OIDC client secret to forgejo from /run, not from the container's disk

Browse source 
swarm-bao-forge-oidc fetched the forge's OIDC client secret from bao and wrote
it to /var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/<id>.secret,
on the container's persistent root, where it outlived a bao rotation and was
in every backup of that tree.

Now the fetch writes it to /run/hive-forge-oidc/<id>.secret on the host
(tmpfs, root 0400). The directory is bound read-only into the container at
the same path, and forgejo-sso-source loads it through LoadCredential. A
tmpfiles rule creates the directory at boot, so the bind source always
exists, and removes the old persistent copy.

After a reboot the file is absent until the fetch succeeds, and
forgejo-sso-source fails to start until then; the fetch starts it again once
it writes the file. Logins keep working meanwhile: forgejo holds the secret
in its login-source row.

deploy.forgejo.sso.clientSecretFile (and its old swarm.forge.sso name) is
removed: it only chose where in the container the fetched file landed. The
fetch no longer needs the container's forgejo uid, so it no longer fails on a
fresh host before the container has created its state dir.

Part of #4998.
This commit is contained in:
atlas 2026-10-15 19:24:23 +02:00 • committed by mara
commit 09ad7386b2
8 changed files with 106 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

121
nix/host-modules/hive-forge/default.nix
View file

@ -28,16 +28,17 @@ let
autheliaUrl = autheliaCfg.url;
autheliaDiscoveryUrl = "${toString autheliaUrl}/.well-known/openid-configuration";
# Where the plaintext lands inside the forge container. Under
# /var/lib rather than /run: the forge may start before the delivery
# unit on a later boot, and a secret that evaporates on reboot turns a
# working login into an intermittent one.
forgeSecretPath = "/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";
# The forge's OIDC client secret, in a host tmpfs directory bound read-only
# into the container at the same path: `swarm-bao-forge-oidc` writes it,
# `forgejo-sso-source` loads it as a credential. After a reboot it is absent
# until the fetch succeeds; logins still work then, because forgejo keeps
# the secret in its login-source row.
oidcSecretDir = "/run/hive-forge-oidc";
oidcSecretPath = "${oidcSecretDir}/${cfg.sso.clientId}.secret";
# The same file seen from the host, where `swarm-bao-forge-oidc` writes it.
# Follows `clientSecretFile`, so an operator-chosen path is where the
# fetched secret lands.
hostSecretPath = "/var/lib/nixos-containers/hive-forge${toString deployCfg.forgejo.sso.clientSecretFile}";
# Where the fetched secret used to be written, on the container's persistent
# root. Removed by the tmpfiles rule below.
oldOidcSecretPath = "/var/lib/nixos-containers/hive-forge/var/lib/forgejo-oidc/${cfg.sso.clientId}.secret";
# Where swarm-secret-publish on authelia's host leaves this client's secret.
# The forge publisher's store role grants read on this one path.
@ -55,8 +56,8 @@ let
swarmControllerForgeUser = "swarm-controller";
# Where the minted token lands inside the forge container — under
# forgejo's own state dir for the same "survives a reboot" reason as
# `forgeSecretPath` above.
# forgejo's own state dir, because the file is the mint's record that it
# has minted and has to survive a reboot.
swarmControllerTokenPath = "/var/lib/forgejo/swarm-controller-token";
# The same file from the host, which is where its publisher reads and watches it.
@ -128,6 +129,11 @@ in
imports = [
./service.nix
(lib.mkRemovedOptionModule [ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ] ''
The forge's OIDC client secret is fetched from the swarm secret store
(swarm-bao-forge-oidc) into a fixed runtime path. Remove this
definition.
'')
(lib.mkRemovedOptionModule
[ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ]
''
@ -249,32 +255,6 @@ in
adds; what `glue-forge-controller.nix` forwards to the controller.
'';
};
# The other half of the SSO pair: the client *id* is swarm-wide (it has
# to match authelia's register), the secret is a path on this host. The
# design note for the login source itself is with the id.
sso.clientSecretFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
example = "/var/lib/forgejo-oidc/forgejo.secret";
description = ''
Path **inside the forge container** holding the client
secret's plaintext.
A path, never a value: an OIDC client secret has two holders
in two containers (authelia keeps a hash, this forge needs the
plaintext), and a literal written here would be rendered into
the world-readable nix store.
Set to `/var/lib/forgejo-oidc/<client id>.secret` wherever the
forge runs. `swarm-bao-forge-oidc` writes the secret here from the
swarm secret store, whichever path this names.
Null is refused at eval: a forge that boots with SSO
half-configured presents as a login button that always fails,
which is harder to diagnose than an eval error.
'';
};
};
config = lib.mkIf deployCfg.forgejo.enable {
@ -371,21 +351,6 @@ in
};
assertions = [
{
# Fail at EVAL, not at boot. The alternative failure is a login
# button that always 401s, three layers from the missing file.
assertion = deployCfg.forgejo.sso.clientSecretFile != null;
message = ''
The forge's SSO login source requires
services.hyperhive.deploy.forgejo.sso.clientSecretFile — the path
(inside the forge container) holding the OIDC client secret's
plaintext.
It is set by default wherever the forge runs, and
swarm-bao-forge-oidc writes the secret there from the swarm
secret store. Remove the definition that set it to null.
'';
}
{
assertion = cfg.rootUrl == null || lib.hasSuffix "/" cfg.rootUrl;
message = ''
@ -460,7 +425,16 @@ in
# delivery. The bind-mount, the `container@` ordering and the bundle
# the container's consumers read all come from the hive-ca-trust
# helper — see the container's `imports` for the consumption half.
bindMounts = caTrust.bindMount;
#
# The OIDC secret's directory: a container's `/run` is an nspawn tmpfs
# the host cannot see into. The host directory comes from the tmpfiles
# rule below, so it exists before the container starts.
bindMounts = caTrust.bindMount // {
${oidcSecretDir} = {
hostPath = oidcSecretDir;
isReadOnly = true;
};
};
config =
{ pkgs, ... }:
let
@ -817,6 +791,10 @@ in
User = "forgejo";
Group = "forgejo";
SyslogIdentifier = "forgejo-sso-source";
# Absent until `swarm-bao-forge-oidc` on the host has fetched
# it, and this unit fails to start until then; that unit starts
# it again once the file is written.
LoadCredential = [ "oidc-client-secret:${oidcSecretPath}" ];
};
# `FORGEJO_CUSTOM` (the forgejo-prefixed name, not `GITEA_CUSTOM`)
# is how the CLI finds the app.ini upstream's module wrote.
@ -842,9 +820,9 @@ in
script = ''
set -euo pipefail
secret=$(cat ${lib.escapeShellArg deployCfg.forgejo.sso.clientSecretFile})
secret=$(cat "$CREDENTIALS_DIRECTORY/oidc-client-secret")
if [ -z "$secret" ]; then
echo "empty OIDC client secret at ${deployCfg.forgejo.sso.clientSecretFile}" >&2
echo "empty OIDC client secret at ${oidcSecretPath}" >&2
exit 1
fi
@ -1008,8 +986,7 @@ in
# The client this login source authenticates as is registered by
# ../glue-forge-oidc-client.nix, wherever authelia runs.
services.hyperhive.deploy.forgejo.sso.clientSecretFile = lib.mkDefault forgeSecretPath;
#
# The forge's OIDC client secret, out of the swarm secret store under the
# forge publisher's leaf — the one route, wherever authelia runs. Same
# shape as ../hive-matrix.nix's `swarm-bao-matrix-oidc`: the login and the
@ -1098,24 +1075,15 @@ in
exit 1
fi
# `forgejo-sso-source` reads the file as forgejo. The owning uid is
# read off the container's forgejo state dir rather than assumed,
# which exists only once the container has started.
state=/var/lib/nixos-containers/hive-forge/var/lib/forgejo
if [ ! -d "$state" ]; then
echo "the forge container has not created $state yet, so the forge's OIDC client secret has no owner to be written for." >&2
exit 1
fi
uid="$(stat -c %u "$state")"
mkdir -p ${lib.escapeShellArg (builtins.dirOf hostSecretPath)}
if secret_differs ${lib.escapeShellArg hostSecretPath} "$secret"; then
atomic_write_secret 0400 "$uid:$uid" ${lib.escapeShellArg hostSecretPath} "$secret"
# Root-owned: the container's systemd reads it for `LoadCredential`.
install -d -m 0700 ${lib.escapeShellArg oidcSecretDir}
if secret_differs ${lib.escapeShellArg oidcSecretPath} "$secret"; then
atomic_write_secret 0400 "" ${lib.escapeShellArg oidcSecretPath} "$secret"
fi
# The login source is a row `forgejo-sso-source` writes from the
# file each time it runs.
refresh_consumer hive-forge forgejo-sso-source.service ${lib.escapeShellArg hostSecretPath}
refresh_consumer hive-forge forgejo-sso-source.service ${lib.escapeShellArg oidcSecretPath}
'';
};
@ -1237,9 +1205,14 @@ in
};
};
# Unused: nothing writes or reads this host path, and it may hold a copy of
# the controller's forge token.
systemd.tmpfiles.rules = [ "r /var/lib/hyperhive-forge/swarm-controller.token" ];
systemd.tmpfiles.rules = [
# The OIDC secret's directory, which the container binds at start.
"d ${oidcSecretDir} 0700 root root -"
# Unused: nothing writes or reads these host paths, and they may hold a
# copy of the controller's forge token or of the forge's OIDC secret.
"r /var/lib/hyperhive-forge/swarm-controller.token"
"r ${oldOidcSecretPath}"
];
networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall {
allowedTCPPorts = [