vinzenz/hyperhive
vinzenz/hyperhive
Watch
0
0
Fork
You've already forked hyperhive
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

nix: run the forge on one host per swarm (deploy.forgejo.enable)

Browse source 
Every hive with hyperhive enabled ran its own hive-forge container, and
its gateway answered forge.<swarm> with its own bridge IP, so on a
multi-host swarm each hive talked to its own forge.

deploy.forgejo.enable defaults to false and allSwarmServices sets it with
mkDefault, like authelia and bao; singleHostSwarm gets it through that.
The forge's OIDC client moves to a glue module gated on authelia, so a
split authelia/forge swarm still registers it. CI now requires the forge
on the same host, and the controller's forgeTokenFile defaults to null
where the forge is not.

Closes #4705
Refs #3782
This commit is contained in:
atlas 2026-09-24 19:44:04 +02:00 • committed by mara
commit 978164dc53
15 changed files with 346 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

4
nix/checks.nix
View file

@ -51,6 +51,10 @@ in
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
module-eval-forge-placement = import ./module-eval/forge-placement.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;
};
module-eval-bao-basics = import ./module-eval/bao-basics.nix {
inherit pkgs self nixosSystem;
inherit (pkgs) lib;

8
nix/host-modules/default.nix
View file

@ -3,10 +3,11 @@
# the package/source wiring; see flake.nix). One import covers
# everything; `services.hyperhive.enable = true` turns the stack on.
#
# The forge is mandatory — hive-c0re mirrors every agent's applied
# Every swarm needs the forge — hive-c0re mirrors every agent's applied
# config repo into it and it's the canonical store for the meta flake
# + `internal/*` repos, so there's no enable toggle; it deploys with
# hyperhive itself. hive-matrix is opt-in (off by default). All
# + `internal/*` repos — but only the host with `deploy.forgejo.enable`
# runs it (derived from `deploy.allSwarmServices`, see
# ./swarm-required-services.nix). hive-matrix is opt-in (off by default). All
# subsystems rely on `services.hyperhive.domain`, which is required
# (asserted in hive-network.nix) whenever hyperhive is enabled.
{
@ -26,6 +27,7 @@
./glue-bao-readers-policy-order.nix
./glue-bao-tls.nix
./glue-controller-bao-identity.nix
./glue-forge-oidc-client.nix
./glue-grafana-oidc-client.nix
./glue-matrix-bao-token.nix
./glue-matrix-ctl-bao-identity.nix

33
nix/host-modules/deploy.nix
View file

@ -418,14 +418,33 @@ in
)
];
# ⚠️ `deploy.forgejo` is declared in ./hive-ci.nix, not here, and it is the
# one entry with no `enable`: the forge is not optional — it is the canonical
# store for the meta flake and every agent's config repo, so it deploys with
# hyperhive itself. Running the CI runner is the only *deployment* decision
# it has, which is exactly the `{ enable; ci; }` shape the header describes,
# minus the half that does not apply. The knobs live with the module that
# reads them; this file stays the registry of toggles.
# ⚠️ Only `deploy.forgejo.enable` is declared here. The rest of
# `deploy.forgejo` is in ./hive-forge/default.nix and ./hive-ci.nix: the
# knobs live with the module that reads them; this file stays the registry
# of toggles.
options.services.hyperhive.deploy = {
forgejo.enable = lib.mkOption {
type = lib.types.bool;
default = false;
example = true;
description = ''
Run the swarm's forge in a `hive-forge` container on this host. A
swarm has one forge, so one host turns this on. It is the
canonical store for the meta flake and every agent's config repo,
so the swarm needs it, but *this* host running it is a decision
like any other shared service's.
With it off, this hive is a *client*: it reaches the swarm's
forge at {option}`services.hyperhive.swarm.forge.domain`, which
the operator's DNS must resolve to the forge's host. No container
is created, and an existing one's state stays on disk under
`/var/lib/nixos-containers/hive-forge/`.
{option}`services.hyperhive.deploy.allSwarmServices` turns it on,
and `singleHostSwarm` through that.
'';
};
grafana.enable = lib.mkOption {
type = lib.types.bool;
default = false;

40
nix/host-modules/glue-forge-oidc-client.nix Normal file
View file

@ -0,0 +1,40 @@
# Glue: register the swarm's forge as an OIDC client wherever authelia runs.
#
# ONE PAIRING PER FILE — forge ← authelia, and nothing else. Deleting this
# leaves a swarm whose forge is not a client authelia has ever heard of, so
# its "sign in with" button can never complete a login and nothing minted its
# secret either.
#
# ⚠️ Gated on authelia being HERE, and deliberately NOT on this host running
# the forge. A client is a row in THIS host's provider config, so it can only be
# declared where that config is rendered — and ./hive-forge/default.nix's whole
# `config` block hangs off `deploy.forgejo.enable`, so a swarm with the forge
# and authelia on different hosts would register the client nowhere at all.
# Same shape as ./glue-grafana-oidc-client.nix, for the same reason.
#
# Unlike Grafana's, this client is never unused: every swarm runs a forge.
{
lib,
config,
...
}:
let
hyperhiveCfg = config.services.hyperhive;
deployCfg = hyperhiveCfg.deploy;
forgeCfg = hyperhiveCfg.swarm.forge;
in
{
config = lib.mkIf (hyperhiveCfg.enable && deployCfg.authelia.enable) {
# One declaration, two readers. The forge knows its own callback URL;
# making the operator restate it in authelia's client list would be a
# second source of truth for a string whose mismatch is a silent
# rejected login.
services.hyperhive.swarm.authelia.oidc.clients = [
{
id = forgeCfg.sso.clientId;
description = "HyperHive forge";
redirectUris = [ forgeCfg.sso.redirectUri ];
}
];
};
}

5
nix/host-modules/glue-grafana-oidc-client.nix
View file

@ -9,9 +9,8 @@
# declared where that config is rendered — and ./swarm-grafana.nix's whole
# `config` block hangs off `deploy.grafana.enable`, so a swarm with Grafana and
# authelia on different hosts registered the client nowhere at all.
# ./hive-forge/default.nix is already on the right side of that line: its module
# is gated on `hyperhive.enable` and only the registration asks about authelia.
# This file puts Grafana there without moving the rest of its module.
# ./glue-forge-oidc-client.nix does the same for the forge. This file puts
# Grafana there without moving the rest of its module.
#
# ⚠️ Registered whether or not the swarm has a Grafana, because nothing in
# `swarm.*` records that — `deploy.grafana.enable` answers "does THIS host run

19
nix/host-modules/hive-ci.nix
View file

@ -82,8 +82,9 @@ in
Run a Forgejo Actions runner in a `hive-ci` nixos-container.
Grouped under `services.hyperhive.deploy.forgejo` because the runner
is tightly coupled to the forge instance it registers against.
Disabled by default; the internal forge it registers against is
always present (mandatory), so enabling this is all that's needed.
Disabled by default. It registers against the forge on this same
host, so it requires
{option}`services.hyperhive.deploy.forgejo.enable` here.
On first start the container auto-registers against hive-forge using
hive-c0re's admin token — no manual token provisioning needed.
@ -170,6 +171,20 @@ in
Set behindGateway = true (it is the default).
'';
}
{
# The runner reaches the forge through THIS host's gateway, and
# hive-c0re registers it through the local forge container; neither
# exists on a host that does not run the forge. A runner on another
# host, registered by the swarm instead, is a separate design.
assertion = forgeDeployCfg.enable;
message = ''
services.hyperhive.deploy.forgejo.ci.enable requires
services.hyperhive.deploy.forgejo.enable on the same host.
The CI runner reaches the forge through this host's gateway and
is registered through the local forge container, so it can only
run where the forge does. Enable CI on the swarm's forge host.
'';
}
];
# The runner's journal, named as the unit is *inside* the container —

60
nix/host-modules/hive-forge/default.nix
View file

@ -105,8 +105,9 @@ let
in
{
# Private Forgejo in a `hive-forge` nixos-container, shared host
# netns. Agents reach it at `forge.<domain>` via the gateway. State
# at `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
# netns, on the one host with `deploy.forgejo.enable`. Agents reach it
# at `forge.<domain>` via the gateway. State at
# `/var/lib/nixos-containers/hive-forge/var/lib/forgejo/` survives
# restart. See `docs/networking/gateway.md::hive-forge container shape`.
# External Forgejo/Gitea/Codeberg-compatible forges (beyond the mandatory
@ -138,10 +139,10 @@ in
'')
];
# The internal forge is mandatory — it's the canonical store for the
# meta flake + every agent's config repo (and the `internal/*` repos),
# so there is no enable/disable toggle. It deploys whenever hyperhive
# itself is enabled (`services.hyperhive.enable`).
# Every swarm needs this forge — it's the canonical store for the meta
# flake + every agent's config repo (and the `internal/*` repos) — but
# only one host runs it: `deploy.forgejo.enable`, in ../deploy.nix. What
# follows is what every hive needs to reach it, wherever it runs.
options.services.hyperhive.swarm.forge = {
httpPort = lib.mkOption {
type = lib.types.port;
@ -252,8 +253,8 @@ in
};
# The swarm's authelia is always registered as an OpenID Connect
# login source here — there is no toggle, for the same reason the
# forge itself has none.
# login source here — there is no toggle: a forge without it has no
# way to log a person in through the swarm's SSO.
#
# **Additive, never exclusive.** Forgejo keeps its local password
# database and gains an extra "sign in with" button; this does not
@ -270,6 +271,29 @@ in
`services.hyperhive.swarm.authelia.oidc.clients`.
'';
};
redirectUri = lib.mkOption {
type = lib.types.str;
readOnly = true;
default = ssoRedirectUri;
defaultText = lib.literalExpression ''"''${ROOT_URL}user/oauth2/authelia/callback"'';
description = ''
OAuth2 callback authelia sends the browser back to, and the URI
it matches **exactly**.
Read-only: forgejo derives it from its own `ROOT_URL` and the
login source's name, so it is a fact other modules read rather
than a knob. The glue that registers this client wherever
authelia runs reads it from here instead of restating the
format.
⚠️ With {option}`services.hyperhive.swarm.forge.rootUrl` unset,
`ROOT_URL` follows
{option}`services.hyperhive.deploy.forgejo.behindGateway` and
the gateway's `httpsPort`, which are per-host. An authelia host
that is not the forge's host renders the forge's callback only
if the two agree on them; set `rootUrl` if they do not.
'';
};
# The secret half is a path on the host that runs the forge, so it
# lives under `deploy.forgejo.sso` — see the block below.
};
@ -435,7 +459,7 @@ in
};
};
config = lib.mkIf config.services.hyperhive.enable {
config = lib.mkIf (config.services.hyperhive.enable && deployCfg.forgejo.enable) {
# Same principle as the vhost below — this service's own surface lives
# with the service. The SSO source is named alongside forgejo because
# its failure mode is a login that silently falls back, not an error.
@ -1194,20 +1218,10 @@ in
};
};
# One declaration, two readers. The forge knows its own callback URL;
# making the operator restate it in authelia's client list would be a
# second source of truth for a string whose mismatch is a silent
# rejected login.
services.hyperhive.swarm.authelia.oidc.clients = lib.mkIf ssoLocal [
{
id = cfg.sso.clientId;
description = "HyperHive forge";
redirectUris = [ ssoRedirectUri ];
}
];
# Same case, same reasoning: this host minted the secret, so it can
# say where the forge will find it.
# The client this login source authenticates as is registered by
# ../glue-forge-oidc-client.nix, wherever authelia runs. When authelia
# is here too, this host minted the secret, so it can say where the
# forge will find it.
services.hyperhive.deploy.forgejo.sso.clientSecretFile = lib.mkIf ssoLocal (
lib.mkDefault forgeSecretPath
);

28
nix/host-modules/swarm-controller.nix
View file

@ -502,16 +502,17 @@ in
forgeTokenFile = lib.mkOption {
type = lib.types.nullOr lib.types.str;
# The forge has no `enable` of its own to condition this on —
# neither half of its split namespace carries one — and it deploys
# unconditionally wherever the rest of the stack does, so its own
# delivery path is simply the right default. The controller's units
# are the thing that decides whether the file is ever read: they only
# exist under `deploy.swarm-controller.enable`, and a host whose forge
# lives elsewhere overrides this (or sets `null`) explicitly.
default = deployCfg.forgejo.hostSwarmControllerTokenFile;
# Forge's own delivery path when the forge runs here, since nothing
# writes that path anywhere else. `null` otherwise, so a controller
# away from the forge logs that it has no forge access instead of
# waiting on a file that never appears. The controller's units are
# the thing that decides whether the file is ever read: they only
# exist under `deploy.swarm-controller.enable`.
default = if deployCfg.forgejo.enable then deployCfg.forgejo.hostSwarmControllerTokenFile else null;
defaultText = lib.literalExpression ''
config.services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile
if config.services.hyperhive.deploy.forgejo.enable
then config.services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile
else null
'';
example = "/var/lib/secrets/swarm-controller-forge.token";
description = ''
@ -520,11 +521,10 @@ in
`forgejo-swarm-controller-account` + `hive-forge-swarm-controller-token`
units, which mint and collect it onto forge's own host).
Defaults to forge's own delivery path (forge deploys
unconditionally alongside the rest of the stack — see
`hive-forge/default.nix`, it has no `enable` of its own).
Override explicitly if forge's actual token file ends up
somewhere else — copy it out of forge's
Defaults to forge's own delivery path when
{option}`services.hyperhive.deploy.forgejo.enable` is set on this
host, and to `null` otherwise. Set it explicitly when the forge
runs on another host — copy the token out of forge's
{option}`services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile`
with whatever secret management this deployment already uses,
the same shape `swarm.nix`'s `clientSecretFile` documents for

24
nix/host-modules/swarm-required-services.nix
View file

@ -4,10 +4,8 @@
# where they live, and asserts the per-service `enable`s that follow —
# the same mode-not-default shape as ./local-defaults.nix, one tier down.
#
# Only the *optional* services derive. The forge has no `enable` to
# assert, because it is not optional — it is the canonical store for the
# meta flake and every agent's config repo, so it deploys with hyperhive
# itself.
# The forge derives from here like the rest: every swarm needs one, but
# only one host in it runs it.
{
lib,
config,
@ -23,20 +21,16 @@ in
example = true;
description = ''
Host the swarm's shared services on this hive. The services that
exist once per swarm rather than once per hive and are *optional*
— the matrix homeserver, the SSO provider, the queue, the metrics
and log stores — have their toggle asserted from this, so a
swarm's service host is declared in one place.
exist once per swarm rather than once per hive — the forge, the
matrix homeserver, the SSO provider, the queue, the metrics and log
stores — have their toggle asserted from this, so a swarm's service
host is declared in one place.
Every toggle it asserts is a {option}`services.hyperhive.deploy.*`
one, because "does THIS host run it" is a per-host decision — which
is the same reason this option is a `deploy.*` one itself. See
./deploy.nix.
The forge is swarm-wide too but has nothing to assert: it is the
canonical store for the meta flake and every agent's config repo,
so it deploys with hyperhive itself and is not optional.
`services.hyperhive.deploy.singleHostSwarm` turns this on as part
of the all-on-one-box mode. Set it directly to run the swarm's
services on a host that is not otherwise all-local — a dedicated
@ -105,4 +99,10 @@ in
# placeable on a host of its own — it can be set directly here and
# turned off wherever this switch happens to be on.
config.services.hyperhive.deploy.bao.enable = lib.mkDefault deployCfg.allSwarmServices;
# The forge. Every swarm needs it, but one host runs it: a hive that does
# not is a *client*, reaching it at `swarm.forge.domain`. Without this a
# `singleHostSwarm` box, which gets here through `allSwarmServices`, would
# have no forge at all.
config.services.hyperhive.deploy.forgejo.enable = lib.mkDefault deployCfg.allSwarmServices;
}

24
nix/module-eval/core-toggle.nix
View file

@ -46,7 +46,11 @@ let
# only hive without one is a hive told to have none.
noAgentQueue = hive { deploy.hive-controller.queue.agentNatsUrl = null; };
withCi = hive { deploy.forgejo.ci.enable = true; };
# On the forge's host: the runner is refused anywhere else.
withCi = hive {
deploy.forgejo.enable = true;
deploy.forgejo.ci.enable = true;
};
# A priority collision is a property of the *option*, not
# of the merged value's interior — nix throws the moment the value is
@ -90,16 +94,16 @@ let
(hive { deploy.forgejo.behindGateway = false; }).services.hyperhive.swarm.forge.publicUrl == null;
}
{
# The controller's token path defaulted to forge's delivery path only on
# a host with the central toggle on, and to `null` otherwise. Forge
# deploys unconditionally, so the path is now unconditional too.
name = "the swarm controller's forgeTokenFile defaults to forge's delivery path regardless of the central toggle";
# The controller's token path follows where the forge runs
# (./forge-placement.nix), never the central toggle: a forge host with
# the toggle off still renders the path.
name = "the swarm controller's forgeTokenFile default does not consult the central toggle";
ok =
let
forgePath = "/var/lib/hyperhive-forge/swarm-controller.token";
in
bare.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath
&& centralToggleOff.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath;
(hive {
enable = false;
deploy.forgejo.enable = true;
}).services.hyperhive.deploy.swarm-controller.forgeTokenFile
== "/var/lib/hyperhive-forge/swarm-controller.token";
}
{
name = "a hive that does not host the swarm's shared services runs none of them";

144
nix/module-eval/forge-placement.nix Normal file
View file

@ -0,0 +1,144 @@
# `checks.module-eval-forge-placement` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
#
# Where the forge runs. A swarm has one, on the host with
# `deploy.forgejo.enable`; every other hive is a client of it, and the parts
# of a split deployment that used to lean on every host running a forge
# (the OIDC client, the controller's token, the CI runner) still have to
# hold.
{
pkgs,
lib,
self,
nixosSystem,
}:
let
inherit
(import ./lib.nix {
inherit
pkgs
lib
self
nixosSystem
;
})
hive
runGroup
;
bare = hive { };
allLocal = hive { deploy.singleHostSwarm = true; };
servicesHere = hive { deploy.allSwarmServices = true; };
forgeHere = hive { deploy.forgejo.enable = true; };
# The shared services here, the forge somewhere else. Every derivation in
# ../host-modules/swarm-required-services.nix is `mkDefault`, so this stays
# expressible — and it is also the authelia-without-forge host the OIDC
# client case needs.
servicesForgeElsewhere = hive {
deploy.allSwarmServices = true;
deploy.forgejo.enable = false;
};
# The forge without authelia: the other half of the split.
forgeNoAuthelia = hive {
deploy.forgejo.enable = true;
deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret";
};
ciNoForge = hive { deploy.forgejo.ci.enable = true; };
ciWithForge = hive {
deploy.forgejo.enable = true;
deploy.forgejo.ci.enable = true;
};
runsForge = m: m.services.hyperhive.deploy.forgejo.enable && m.containers ? hive-forge;
forgeClientIds =
m:
map (c: c.id) (
lib.filter (
c: c.id == m.services.hyperhive.swarm.forge.sso.clientId
) m.services.hyperhive.swarm.authelia.oidc.clients
);
# Matched on the option the message names, same reasoning as
# ./grafana.nix's `grafanaRefusedFor`.
refusedOver = m: needle: lib.any (a: !a.assertion && lib.hasInfix needle a.message) m.assertions;
tokenFile = m: m.services.hyperhive.deploy.swarm-controller.forgeTokenFile;
forgePath = "/var/lib/hyperhive-forge/swarm-controller.token";
cases = [
{
# The absence the whole option exists for: a second forge in a swarm
# is a split brain nobody notices, so a hive that has not been told it
# is the forge's host runs none of its surface.
name = "a hive that is not the forge's host runs no forge";
ok =
!bare.services.hyperhive.deploy.forgejo.enable
&& !(bare.containers ? hive-forge)
&& !(bare.systemd.services ? hive-forge-swarm-controller-token)
&& !(lib.elem "forge.t.local" bare.services.hyperhive.gateway.localNames)
&& !(refusedOver bare "deploy.forgejo.sso.clientSecretFile");
}
{
name = "hosting the swarm's shared services runs the forge";
ok = runsForge servicesHere;
}
{
name = "the all-local mode runs the forge";
ok = runsForge allLocal && lib.elem "forge.t.local" allLocal.services.hyperhive.gateway.localNames;
}
{
name = "an explicit deploy.forgejo.enable runs the forge on its own";
ok = runsForge forgeHere && !forgeHere.services.hyperhive.deploy.allSwarmServices;
}
{
# `mkDefault`, not a plain assignment: the forge stays placeable on a
# host of its own. `nats` is the control, so the case cannot pass on a
# fixture where nothing came on.
name = "placing the forge elsewhere survives the switch that would enable it";
ok =
!(servicesForgeElsewhere.containers ? hive-forge)
&& servicesForgeElsewhere.services.hyperhive.deploy.nats.enable;
}
{
# A client is a row in authelia's config, so it is declared where
# authelia runs. With the forge's module gated, registering it from
# there would leave a split swarm's forge unknown to its IdP.
name = "the forge's OIDC client is registered wherever authelia runs, and only there";
ok =
forgeClientIds servicesForgeElsewhere == [ "forgejo" ]
&& forgeClientIds allLocal == [ "forgejo" ]
&& forgeClientIds forgeNoAuthelia == [ ];
}
{
name = "the forge's OIDC callback is the one forgejo sends";
ok =
servicesForgeElsewhere.services.hyperhive.swarm.forge.sso.redirectUri
== "https://forge.t.local/user/oauth2/authelia/callback";
}
{
# The runner reaches the forge through this host's gateway and is
# registered through the local container. The second arm is the
# control: a refusal that fires everywhere is not a check.
name = "CI is refused on a host that does not run the forge";
ok =
refusedOver ciNoForge "deploy.forgejo.enable on the same host"
&& !(refusedOver ciWithForge "deploy.forgejo.enable on the same host");
}
{
# Nothing writes the delivery path away from the forge, and a
# `LoadCredential=` naming a missing path is fatal to the unit.
name = "the controller's forge token defaults to the delivery path only where the forge runs";
ok =
tokenFile bare == null
&& tokenFile servicesForgeElsewhere == null
&& tokenFile forgeHere == forgePath
&& tokenFile allLocal == forgePath;
}
];
in
runGroup "forge-placement" cases