From 368f5d82aa1bdd112777d1b1dcf732373bcd826e Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 00:54:45 +0200 Subject: [PATCH 1/6] deploy: move the wireguard mesh out of the namespace hives read `swarm.*` is what a hive needs to be a *client* of the swarm; the mesh is none of it. A peer needs this host's `wireguardEndpoint` -- the roster entry in swarm.nix, which stays -- and nothing about the interface this host brings up. The module already said so: "plain host networking that a machine which runs no hive at all still needs." All five options move, so the namespace relocates rather than splitting. `listenPort` is the one that reads the other way: it is what this host *binds*, while the port a peer *dials* lives inside `wireguardEndpoint`. Declared in swarm-wireguard.nix under the `deploy.*` path, following swarm-victorialogs.nix; deploy.nix carries only the renames, per its own "a single file to delete when the deprecation window closes". Deliberately NOT added to deploy.nix's own options block: every entry there is a swarm service this host deploys, and the mesh is host networking. hivectl/src/wg.rs generates the config snippet an operator pastes, so it moves too -- otherwise the tool's own output trips the deprecation warning. module-eval gains a case that configures a host through the OLD path and asserts the rendered wg-hive interface, because the new path evaluates fine without the shim: dropping it reads as a clean tree. --- docs/networking/snapshot-store.md | 4 +-- docs/swarm/README.md | 6 ++--- docs/swarm/secrets.md | 2 +- hivectl/src/wg.rs | 6 ++--- nix/host-modules/deploy.nix | 28 +++++++++++++++++++++ nix/host-modules/swarm-snapshot-store.nix | 10 ++++---- nix/host-modules/swarm-wireguard.nix | 30 ++++++++++++++--------- nix/host-modules/swarm.nix | 2 +- nix/module-eval.nix | 22 +++++++++++++++++ nix/packages/default.nix | 2 +- 10 files changed, 85 insertions(+), 27 deletions(-) diff --git a/docs/networking/snapshot-store.md b/docs/networking/snapshot-store.md index efe30c33..2c71e9b3 100644 --- a/docs/networking/snapshot-store.md +++ b/docs/networking/snapshot-store.md @@ -26,7 +26,7 @@ services.hyperhive.snapshotStore = { }; # The mesh is a hard requirement, and is asserted: -services.hyperhive.swarm.wireguard = { +services.hyperhive.deploy.wireguard = { enable = true; address = "10.100.0.9/24"; privateKeyFile = "/etc/wireguard/hive.key"; @@ -37,7 +37,7 @@ The store host is a swarm member like any other: it gets an entry in `services.hyperhive.swarm.hives`, the same directory every host holds. See [swarm/](../swarm/README.md) for the mesh itself. -Note that the mesh is gated on `swarm.wireguard.enable`, **not** on +Note that the mesh is gated on `deploy.wireguard.enable`, **not** on `c0re.enable` --- a store host runs no hive and would otherwise get no `wg-hive` interface at all. diff --git a/docs/swarm/README.md b/docs/swarm/README.md index 88ea7d3b..2729f9d1 100644 --- a/docs/swarm/README.md +++ b/docs/swarm/README.md @@ -168,7 +168,7 @@ evaluates cleanly points at a real machine that isn't the one you meant. issue. Until then, federation needs CA-issued certs (ACME). See `docs/integrations/matrix.md` for federation firewall + TLS requirements. -3. **WireGuard mesh** (optional) — `swarm.wireguard.enable` reads each +3. **WireGuard mesh** (optional) — `deploy.wireguard.enable` reads each entry's `wireguardPublicKey`/`wireguardEndpoint`/`wireguardAddress` to configure `wg-hive`. See "WireGuard inter-hive mesh" below. @@ -210,7 +210,7 @@ wg pubkey < /etc/wireguard/hive.key # → share this with peer operators ```nix # hive A (pr1ma.example.com, mesh IP 10.100.0.1) services.hyperhive = { - swarm.wireguard = { + deploy.wireguard = { enable = true; privateKeyFile = "/etc/wireguard/hive.key"; address = "10.100.0.1/24"; @@ -237,7 +237,7 @@ services.hyperhive = { # hive B (edge.corp, mesh IP 10.100.0.2) services.hyperhive = { - swarm.wireguard = { + deploy.wireguard = { enable = true; privateKeyFile = "/etc/wireguard/hive.key"; address = "10.100.0.2/24"; diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 7d1acf16..4c1cdd4a 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -53,7 +53,7 @@ neither is a renaming of the other. | OIDC client secret, digest half | the same mint | `oidc-clients/.digest` | authelia's own half; merged at runtime via `settingsFiles` | | the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `swarm.otel.clientSecretFile` — the collector need not share a host with authelia | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | -| wireguard private key | **the operator** — `wg genkey` | whatever `swarm.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | +| wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | | queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | | the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | | the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it is stated rather than inferred | n/a — only the host running the store seals anything | diff --git a/hivectl/src/wg.rs b/hivectl/src/wg.rs index 272e1e05..ab86431d 100644 --- a/hivectl/src/wg.rs +++ b/hivectl/src/wg.rs @@ -9,7 +9,7 @@ use anyhow::{Context as _, Result, bail}; use crate::util::query_hive_urls; /// Host path of this hive's WireGuard private key (matches the -/// `privateKeyFile` example in the swarm.wireguard nix options). +/// `privateKeyFile` example in the deploy.wireguard nix options). const WG_KEY_PATH: &str = "/etc/wireguard/hive.key"; /// The mesh interface name hive-c0re's nix module brings up. const WG_INTERFACE: &str = "wg-hive"; @@ -116,7 +116,7 @@ pub(crate) async fn wg_init(socket: &Path, address: Option<&str>) -> Result<()> println!("\nPublic key (share this with peer hives — they pass it to `hivectl wg peer`):"); println!(" {pubkey}"); println!("\nAdd to this hive's NixOS config:"); - println!(" services.hyperhive.swarm.wireguard = {{"); + println!(" services.hyperhive.deploy.wireguard = {{"); println!(" enable = true;"); println!(" privateKeyFile = \"{WG_KEY_PATH}\";"); println!(" address = \"{addr}\";"); @@ -238,7 +238,7 @@ pub(crate) fn wg_status() -> Result<()> { if !status.success() { bail!( "`wg show {WG_INTERFACE}` failed — is the mesh enabled + up? \ - (services.hyperhive.swarm.wireguard.enable = true, then deploy)" + (services.hyperhive.deploy.wireguard.enable = true, then deploy)" ); } Ok(()) diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index a2551a1a..29e0171c 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -148,6 +148,34 @@ in [ "services" "hyperhive" "tls" "leafValidityDays" ] [ "services" "hyperhive" "deploy" "hive-controller" "tls" "leafValidityDays" ] ) + + # The WireGuard mesh, whole. Unlike every rename above this one moves a + # namespace rather than a toggle: nothing under it is a fact another hive + # reads. A peer needs this host's `wireguardEndpoint` — the roster entry + # in ./swarm.nix, which stays — and nothing about the interface this host + # brings up. `listenPort` moves for the same reason and is easy to read + # the other way: it is what this host *binds*, while the port a peer + # *dials* is the one inside `wireguardEndpoint`. + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "wireguard" "enable" ] + [ "services" "hyperhive" "deploy" "wireguard" "enable" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "wireguard" "privateKeyFile" ] + [ "services" "hyperhive" "deploy" "wireguard" "privateKeyFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "wireguard" "address" ] + [ "services" "hyperhive" "deploy" "wireguard" "address" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "wireguard" "listenPort" ] + [ "services" "hyperhive" "deploy" "wireguard" "listenPort" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "wireguard" "persistentKeepalive" ] + [ "services" "hyperhive" "deploy" "wireguard" "persistentKeepalive" ] + ) ]; # ⚠️ `deploy.forgejo` is declared in ./hive-ci.nix, not here, and it is the diff --git a/nix/host-modules/swarm-snapshot-store.nix b/nix/host-modules/swarm-snapshot-store.nix index 24694ab9..4ad556ff 100644 --- a/nix/host-modules/swarm-snapshot-store.nix +++ b/nix/host-modules/swarm-snapshot-store.nix @@ -26,9 +26,9 @@ }: let cfg = config.services.hyperhive.snapshotStore; - wgCfg = config.services.hyperhive.swarm.wireguard; + wgCfg = config.services.hyperhive.deploy.wireguard; - # `swarm.wireguard.address` carries a prefix ("10.100.0.1/24") because + # `deploy.wireguard.address` carries a prefix ("10.100.0.1/24") because # it feeds `networking.wireguard.interfaces.wg-hive.ips`. A listen # address must be the bare IP, so strip it. meshAddress = lib.head (lib.splitString "/" wgCfg.address); @@ -103,7 +103,7 @@ in mesh. Off by default --- it is a distinct deployment role, not part of a hive. - Requires `services.hyperhive.swarm.wireguard.enable`: the mesh + Requires `services.hyperhive.deploy.wireguard.enable`: the mesh is both the transport and the authentication, so there is no meaningful configuration without it. ''; @@ -138,7 +138,7 @@ in assertion = wgCfg.enable; message = '' services.hyperhive.snapshotStore.enable requires - services.hyperhive.swarm.wireguard.enable --- the mesh is the + services.hyperhive.deploy.wireguard.enable --- the mesh is the store's transport AND its authentication (cryptokey routing binds a peer's source address to its public key). Without it there is nothing to bind the listener to and no way to tell @@ -149,7 +149,7 @@ in assertion = wgCfg.address != ""; message = '' services.hyperhive.snapshotStore.enable requires - services.hyperhive.swarm.wireguard.address to be set --- the + services.hyperhive.deploy.wireguard.address to be set --- the receiver binds to this host's mesh address, and refuses to fall back to a wildcard. ''; diff --git a/nix/host-modules/swarm-wireguard.nix b/nix/host-modules/swarm-wireguard.nix index e6d982c7..7b2e1694 100644 --- a/nix/host-modules/swarm-wireguard.nix +++ b/nix/host-modules/swarm-wireguard.nix @@ -8,6 +8,14 @@ # The two stay coupled by data, not by structure: the per-peer # `wireguard*` fields live on the peer submodule in ./swarm.nix, since # that is where a peer is described, and this module reads them. +# +# Everything declared here is `deploy.*`, not `swarm.*`: by ./deploy.nix's +# rule, `swarm.*` is what a hive needs to be a *client* of the swarm, and +# none of this is. A peer needs this host's `wireguardEndpoint` (the roster +# entry in ./swarm.nix); what interface this host brings up, on which +# address, with which key, is nobody else's business. `listenPort` included +# — it is what this host binds, while the port a peer dials is the one +# inside `wireguardEndpoint`. { lib, config, @@ -19,7 +27,7 @@ # `wireguardPublicKey` declared. Peers reachable over the mesh are # preferred for inter-hive traffic (no public TLS round-trip needed); # peers without a public key still work via normal HTTPS. - options.services.hyperhive.swarm.wireguard = { + options.services.hyperhive.deploy.wireguard = { enable = lib.mkOption { type = lib.types.bool; default = false; @@ -39,7 +47,7 @@ Path to the host's WireGuard private key file. The file must be readable by root and should have mode 0400. Generate with `wg genkey > /etc/wireguard/hive.key`. Required when - `swarm.wireguard.enable = true`. + `deploy.wireguard.enable = true`. ''; }; @@ -83,21 +91,21 @@ # at all. Nothing below is c0re-specific; the peer data c0re consumes # (HIVE_PEER_CA_PATHS) is rendered in ./hive-c0re and stays gated # there. - config = lib.mkIf config.services.hyperhive.swarm.wireguard.enable { + config = lib.mkIf config.services.hyperhive.deploy.wireguard.enable { assertions = [ { - assertion = config.services.hyperhive.swarm.wireguard.privateKeyFile != null; + assertion = config.services.hyperhive.deploy.wireguard.privateKeyFile != null; message = '' - services.hyperhive.swarm.wireguard.enable requires - services.hyperhive.swarm.wireguard.privateKeyFile to be set. + services.hyperhive.deploy.wireguard.enable requires + services.hyperhive.deploy.wireguard.privateKeyFile to be set. Generate a key: wg genkey > /etc/wireguard/hive.key ''; } { - assertion = config.services.hyperhive.swarm.wireguard.address != ""; + assertion = config.services.hyperhive.deploy.wireguard.address != ""; message = '' - services.hyperhive.swarm.wireguard.enable requires - services.hyperhive.swarm.wireguard.address to be set + services.hyperhive.deploy.wireguard.enable requires + services.hyperhive.deploy.wireguard.address to be set (e.g. "10.100.0.1/24"). ''; } @@ -107,7 +115,7 @@ # connects to each peer that has `wireguardPublicKey` set. networking.wireguard.interfaces = let - wgCfg = config.services.hyperhive.swarm.wireguard; + wgCfg = config.services.hyperhive.deploy.wireguard; # `peerHives` is `swarm.hives` minus this hive (../swarm.nix) — # a mesh that included our own entry would configure a tunnel to # ourselves. @@ -139,7 +147,7 @@ # Open the WireGuard UDP port on the host firewall (host-level # networking — not inside containers). networking.firewall.allowedUDPPorts = [ - config.services.hyperhive.swarm.wireguard.listenPort + config.services.hyperhive.deploy.wireguard.listenPort ]; }; } diff --git a/nix/host-modules/swarm.nix b/nix/host-modules/swarm.nix index 0b1016b4..bf0a73cc 100644 --- a/nix/host-modules/swarm.nix +++ b/nix/host-modules/swarm.nix @@ -144,7 +144,7 @@ in example = "base64pubkey="; description = '' WireGuard public key for this hive's host. Required when - `services.hyperhive.swarm.wireguard.enable = true` and + `services.hyperhive.deploy.wireguard.enable = true` and you want this hive reachable over the mesh. Null = TLS- only peering (public internet, no mesh tunnel). ''; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 49678d98..396d356a 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -62,6 +62,16 @@ let bare = hive { }; withCi = hive { deploy.forgejo.ci.enable = true; }; + # A host configured against the pre-rename option path. `mkRenamedOptionModule` + # is the only thing carrying it, and nothing else in this suite would notice + # if it were dropped: the new path evaluates fine on its own, so a missing + # shim reads as a clean tree and breaks every existing operator config. + wireguardOldPath = hive { + swarm.wireguard.enable = true; + swarm.wireguard.address = "10.100.0.1/24"; + swarm.wireguard.privateKeyFile = "/etc/wireguard/hive.key"; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -183,6 +193,18 @@ let name = "the all-local mode turns the swarm controller on"; ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; } + { + # The mesh moved namespace wholesale, so an existing config sets paths + # that no longer exist. Reading the *rendered interface* rather than the + # option: a rename that resolved but stopped reaching the module would + # satisfy an option-level check and still bring up no tunnel. + name = "a config written against the pre-rename wireguard path still configures the interface"; + ok = + let + wg = wireguardOldPath.networking.wireguard.interfaces.wg-hive; + in + wg.ips == [ "10.100.0.1/24" ] && wg.privateKeyFile == "/etc/wireguard/hive.key"; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 9f673613..d3241997 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -93,7 +93,7 @@ let # Completions come from the binary's own `completions ` verb # (the single source of truth, so they never drift from the actual # verbs). The wireguard-tools wrap makes `hivectl wg` subcommands - # work before `swarm.wireguard.enable` is set (wg init is the very + # work before `deploy.wireguard.enable` is set (wg init is the very # first setup step). Completion generation runs before wrapProgram # since wrapProgram renames the real binary. hivectlPkg = From 1db2ac26a799da9917cbdecbc58fb6636c36023c Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 01:38:53 +0200 Subject: [PATCH 2/6] deploy: split the forge's host decisions out of swarm.forge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swarm.*` is what a hive needs to be a *client* of the swarm. For the forge that is what it IS from any hive's point of view: its package, the names and ports it answers on, the URLs it advertises, and the client id it is registered under. How it is served, what it mirrors and where its host-local secrets sit are decisions of the machine running it, so behindGateway, openFirewall, mirrors, sso.clientSecretFile and hostSwarmControllerTokenFile move to `deploy.forgejo.*`. Unlike the wireguard mesh this SPLITS a module rather than relocating a whole namespace. `sso` splits with it: `clientId` stays because it must match the id in authelia's register, while the secret beside it is a path on one host. Moving the whole `sso` block for symmetry with `ci` was considered and rejected on exactly that asymmetry. Declared in hive-forge/default.nix under the `deploy.*` path, following swarm-victorialogs.nix; deploy.nix carries only the renames. `mirrors` renames in one entry rather than one per field — it is a single option of a list-of-submodule type, so the rename carries its whole value, where `ci` needed five because it is a plain attrset of options. Readers outside the module: hive-ci.nix binds `deploy.forgejo` for its behindGateway assertion; swarm-authelia.nix and swarm-controller.nix read theirs off the `deployCfg` they already bind. hivectl's `open` printed `services.hyperhive.forge.behindGateway` in an operator-facing hint — a path that never existed, missing `swarm.` — and hive-c0re's state_snapshot doc comment carried the same defect; both now name the new path. The rendered docs put the two halves on separate pages, so the five descriptions of staying options that explain themselves in terms of `behindGateway` now qualify it in full. module-eval gains a forge case configured entirely through the old paths, asserting the rendered firewall ports and the mirror env var c0re seeds from: the new paths evaluate fine without the shims, so dropping them reads as a clean tree. All five old paths are defined in the fixture, so removing any single shim entry fails the eval rather than only the two the assertion reads. --- docs/integrations/forge.md | 6 +- docs/networking/gateway.md | 20 ++- docs/scheduler/ci.md | 6 +- docs/tools/hivectl.md | 2 +- hive-c0re/src/dashboard/state_snapshot.rs | 4 +- hivectl/src/open.rs | 2 +- nix/host-modules/deploy.nix | 32 ++++ nix/host-modules/hive-c0re/environment.nix | 4 +- nix/host-modules/hive-ci.nix | 17 +- nix/host-modules/hive-forge/default.nix | 194 +++++++++++---------- nix/host-modules/swarm-authelia.nix | 2 +- nix/host-modules/swarm-controller.nix | 9 +- nix/module-eval.nix | 31 ++++ 13 files changed, 209 insertions(+), 120 deletions(-) diff --git a/docs/integrations/forge.md b/docs/integrations/forge.md index 10c15f01..26e899ad 100644 --- a/docs/integrations/forge.md +++ b/docs/integrations/forge.md @@ -2,7 +2,11 @@ Private Forgejo instance running in a nixos-container, used as the swarm's persistent code-collaboration surface (issues, PRs, reviews, -attachments). Configured via `services.hyperhive.swarm.forge.*`. Container +attachments). Configured via `services.hyperhive.swarm.forge.*` (what the +forge IS to every hive — its package, names, ports, URLs) plus +`services.hyperhive.deploy.forgejo.*` (what the host running it decides — +how it is served, what it mirrors, where its host-local secrets sit). +Container shape, ROOT_URL / sub-domain routing, and operator-vs-in-cluster URL handling live in [`docs/networking/gateway.md`](../networking/gateway.md); this file owns the per-agent integration story and the notification pump that wakes diff --git a/docs/networking/gateway.md b/docs/networking/gateway.md index a5aa5c75..338a019e 100644 --- a/docs/networking/gateway.md +++ b/docs/networking/gateway.md @@ -10,7 +10,7 @@ Single nginx in front of every hyperhive web surface. Runs on the **host**, next | `/agent//` | `_` | per-agent harness (UDS or TCP) | `agents.conf` (runtime-generated) | | `/.well-known/matrix/{client,server}` | `_` | inline JSON (no upstream) | `matrix.enable && domain != null` | | `/matrix/` (deprecated) | `_` | 301 → `chat./` | `matrix.gui.enable` | -| `forge./` | `forge.` | forgejo (`3000`) | `forge.behindGateway` | +| `forge./` | `forge.` | forgejo (`3000`) | `deploy.forgejo.behindGateway` | | `chat./_matrix/*` | `chat.` | tuwunel (`8008`) | `matrix.gatewayHost != null` | | `chat./` | `chat.` | fluffychat-web static | `matrix.gui.enable` | | `chat./config.json` | `chat.` | inline JSON (FluffyChat boot config) | `matrix.gui.enable && domain != null` | @@ -62,7 +62,7 @@ Each location carries a duplicated `auth_basic` block (separate locations don't `services.hyperhive.gateway.localHostsEntry = true` adds entries to the host's `/etc/hosts`: - `` → `127.0.0.1` -- `forge.` → `127.0.0.1` (when forge.behindGateway) +- `forge.` → `127.0.0.1` (when deploy.forgejo.behindGateway) - `chat.` → `127.0.0.1` (when matrix.gatewayHost set) - `auth.` → `127.0.0.1` (when deploy.authelia) @@ -159,7 +159,7 @@ flip together: the primary agent-name link, the favicon fetch (`/icon`), and the nav-strip `container`-kind links from `DashboardState.links` (`GET /api/dashboard-state`). `forge`-kind nav-strip links still resolve against `http://:3000` (separate sub-domain transition -tracked by `forge.behindGateway`); `external`-kind links are +tracked by `deploy.forgejo.behindGateway`); `external`-kind links are already absolute. See `docs/web-ui/dashboard.md::Container row` for the frontend-side derivation. @@ -333,10 +333,12 @@ container. ```nix services.hyperhive.swarm.forge = { - httpPort = 3000; # default — HTTP listener; outside hyperhive's 7000/8100-8999 range - sshPort = 2222; # default — git-over-SSH; kept off 22 so it doesn't collide with the host openssh - openFirewall = false; # default — expose httpPort + sshPort to the host firewall + httpPort = 3000; # default — HTTP listener; outside hyperhive's 7000/8100-8999 range + sshPort = 2222; # default — git-over-SSH; kept off 22 so it doesn't collide with the host openssh }; +# Which ports the forge answers on is swarm-wide; whether THIS host opens +# them in its firewall is a deployment decision, so it lives under deploy.*. +services.hyperhive.deploy.forgejo.openFirewall = false; # default ``` `httpPort` (default **3000**) is the port Forgejo's HTTP server binds to. @@ -357,7 +359,7 @@ the bridge), not the raw port, so no firewall hole is needed. Flip to (not behind the gateway). - External git clients that push/pull via SSH directly to the host. -Forgejo served through the gateway (`forge.behindGateway = true`) does +Forgejo served through the gateway (`deploy.forgejo.behindGateway = true`) does not need `openFirewall` — the gateway's own `openFirewall` option covers that path. @@ -373,8 +375,8 @@ covers most cases: | Shape | Auto-derived `ROOT_URL` | |---|---| -| `behindGateway = true` | `https:///` (port suffix omitted when `gateway.httpsPort == 443`) | -| `behindGateway = false` | `http://:/` | +| `deploy.forgejo.behindGateway = true` | `https:///` (port suffix omitted when `gateway.httpsPort == 443`) | +| `deploy.forgejo.behindGateway = false` | `http://:/` | The gateway always terminates TLS, so the `behindGateway = true` case is always advertised over `https://`; only the direct (`behindGateway = diff --git a/docs/scheduler/ci.md b/docs/scheduler/ci.md index 6d1ced37..2640a155 100644 --- a/docs/scheduler/ci.md +++ b/docs/scheduler/ci.md @@ -100,7 +100,7 @@ build can't hold the runner's single slot indefinitely). ## Container design -- **Private netns, bridge-attached**: the container runs in its own network namespace (`privateNetwork = true`, `hostBridge`) and reaches hive-forge through the gateway at `http://` (resolved to the bridge IP via `networking.extraHosts`). It cannot reach host-loopback services — the core dashboard at `127.0.0.1:7000` and the raw forge port are unreachable from CI. Requires `forge.behindGateway = true`. +- **Private netns, bridge-attached**: the container runs in its own network namespace (`privateNetwork = true`, `hostBridge`) and reaches hive-forge through the gateway at `http://` (resolved to the bridge IP via `networking.extraHosts`). It cannot reach host-loopback services — the core dashboard at `127.0.0.1:7000` and the raw forge port are unreachable from CI. Requires `deploy.forgejo.behindGateway = true`. - **Non-ephemeral**: runner credentials persist across restarts (written to container's stateDir on first registration, reused thereafter). - **Sandbox fallback**: nspawn containers can't create user-namespaces, so nix's sandboxing would always fail. Module sets `nix.settings.sandbox-fallback = true` in the container — nix builds run unsandboxed (safe because the container is already isolated). See `docs/process/gotchas.md`. - **Credential isolation**: the forge admin token (`forge-core-token`) never enters the container. hive-c0re holds it and performs all forge API calls (runner validation + registration-token mint, in `forge/ci_runner.rs`); via hive-priv it writes only the runner registration token to the host env-file `/run/hive-ci/runner-token`, which the container bind-mounts read-only. @@ -150,10 +150,10 @@ forge provisioning sweep (orgs, agent accounts, etc.), so mirror seeding lives in the same place rather than a separate host-side unit. **General-purpose mirrors**: you can pre-seed any external repo as a -pull-mirror via `services.hyperhive.swarm.forge.mirrors`: +pull-mirror via `services.hyperhive.deploy.forgejo.mirrors`: ```nix -services.hyperhive.swarm.forge.mirrors = [ +services.hyperhive.deploy.forgejo.mirrors = [ { upstream = "https://github.com/actions/checkout"; dest = "actions/checkout"; } { upstream = "https://github.com/example/tool"; dest = "mirrors/tool"; } ]; diff --git a/docs/tools/hivectl.md b/docs/tools/hivectl.md index c701e39e..d985d582 100644 --- a/docs/tools/hivectl.md +++ b/docs/tools/hivectl.md @@ -314,6 +314,6 @@ an error. A surface has no URL when it isn't browser-reachable: `home` needs `services.hyperhive.domain`; `forge` needs -`services.hyperhive.swarm.forge.behindGateway = true`; `matrix` needs +`services.hyperhive.deploy.forgejo.behindGateway = true`; `matrix` needs `services.hyperhive.swarm.matrix.gui.enable = true`. In those cases the command exits with a hint naming the option to set. diff --git a/hive-c0re/src/dashboard/state_snapshot.rs b/hive-c0re/src/dashboard/state_snapshot.rs index 5b7fc68b..be12d782 100644 --- a/hive-c0re/src/dashboard/state_snapshot.rs +++ b/hive-c0re/src/dashboard/state_snapshot.rs @@ -94,8 +94,8 @@ pub(super) struct StateSnapshot { /// Public URL of the forge (e.g. /// `"https://forge.pr1ma.darkest.space"`). Sourced from the /// `HIVE_FORGE_PUBLIC_URL` env var, which the c0re NixOS module - /// sets from `services.hyperhive.forge.publicUrl` (defaults to - /// the gateway vhost URL when `forge.behindGateway = true`, + /// sets from `services.hyperhive.swarm.forge.publicUrl` (defaults to + /// the gateway vhost URL when `deploy.forgejo.behindGateway = true`, /// `null` otherwise). `None` when absent — the frontend **hides** /// forge links rather than guessing `http://:3000`, /// which is only right by accident on deployments that aren't diff --git a/hivectl/src/open.rs b/hivectl/src/open.rs index c497cbf0..16e366dd 100644 --- a/hivectl/src/open.rs +++ b/hivectl/src/open.rs @@ -26,7 +26,7 @@ pub(crate) async fn open_url(socket: &Path, target: OpenTarget) -> Result<()> { ), OpenTarget::Forge => ( urls.forge, - "the public forge URL needs `services.hyperhive.forge.behindGateway = true`", + "the public forge URL needs `services.hyperhive.deploy.forgejo.behindGateway = true`", ), OpenTarget::Matrix => ( urls.matrix, diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 29e0171c..78d74f71 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -98,6 +98,38 @@ in [ "services" "hyperhive" "deploy" "forgejo" "ci" "package" ] ) + # The rest of the forge split. What stays under `swarm.forge` is what the + # forge IS from any hive's point of view — its package, the names and ports + # it answers on, the URLs it advertises, the client id it is registered + # under; these five are what the host running it decides. `sso` splits + # because its two halves are different facts: the client id must match the + # entry in authelia's register, the secret is a path on this machine. + # + # `mirrors` renames in one entry rather than one per field: it is a single + # option of a list-of-submodule type, so the rename carries its whole + # value. The `ci` block above needs five because it is a plain attrset of + # separate options, which is the case with no parent path to rename. + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "forge" "behindGateway" ] + [ "services" "hyperhive" "deploy" "forgejo" "behindGateway" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "forge" "openFirewall" ] + [ "services" "hyperhive" "deploy" "forgejo" "openFirewall" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "forge" "mirrors" ] + [ "services" "hyperhive" "deploy" "forgejo" "mirrors" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "forge" "sso" "clientSecretFile" ] + [ "services" "hyperhive" "deploy" "forgejo" "sso" "clientSecretFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "forge" "hostSwarmControllerTokenFile" ] + [ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ] + ) + # Retention is read only where the container is defined, so it is a # decision of the host running the store rather than something the swarm # agrees on. The two stores keep everything else — package, domain, port diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 8539e6b4..9882bead 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -220,8 +220,8 @@ in # breaks the moment the operator's browser hostname isn't the forge # host, e.g. through the gateway or a reverse proxy). Sourced from # `services.hyperhive.swarm.forge.publicUrl`, which itself defaults to the - # gateway vhost URL when `behindGateway = true` and `null` otherwise - # — see that option's doc for the "hide, don't guess" rationale. + # gateway vhost URL when `deploy.forgejo.behindGateway = true` and `null` + # otherwise — see that option's doc for the "hide, don't guess" rationale. # Absent here whenever `publicUrl` is `null`; the dashboard hides # forge links rather than emitting one it can't justify. HIVE_FORGE_PUBLIC_URL = config.services.hyperhive.swarm.forge.publicUrl; diff --git a/nix/host-modules/hive-ci.nix b/nix/host-modules/hive-ci.nix index e52f8adb..1e41285d 100644 --- a/nix/host-modules/hive-ci.nix +++ b/nix/host-modules/hive-ci.nix @@ -7,6 +7,7 @@ let cfg = config.services.hyperhive.deploy.forgejo.ci; forgeCfg = config.services.hyperhive.swarm.forge; + forgeDeployCfg = config.services.hyperhive.deploy.forgejo; gatewayCfg = config.services.hyperhive.gateway; networkCfg = config.services.hyperhive.network; tlsCfg = config.services.hyperhive.deploy.hive-controller.tls; @@ -79,8 +80,8 @@ in example = true; description = '' Run a Forgejo Actions runner in a `hive-ci` nixos-container. - Grouped under `services.hyperhive.swarm.forge` because the runner is - tightly coupled to the forge instance it registers against. + Grouped under `services.hyperhive.deploy.forgejo` because the runner + is tightly coupled to the forge instance it registers against. Disabled by default; the internal forge it registers against is always present (mandatory), so enabling this is all that's needed. @@ -153,17 +154,17 @@ in }; config = lib.mkIf cfg.enable { - # `forge.behindGateway = true` (the default) is required because the - # CI container uses private networking and reaches the forge through + # `deploy.forgejo.behindGateway = true` (the default) is required because + # the CI container uses private networking and reaches the forge through # the gateway vhost. Without the gateway vhost there is no HTTP # listener for `forgeCfg.domain` on the bridge that the runner can # connect to. assertions = [ { - assertion = forgeCfg.behindGateway; + assertion = forgeDeployCfg.behindGateway; message = '' services.hyperhive.deploy.forgejo.ci.enable requires - services.hyperhive.swarm.forge.behindGateway = true. + services.hyperhive.deploy.forgejo.behindGateway = true. The CI container runs with a private network namespace and reaches the forge through the gateway vhost on the bridge IP. Set behindGateway = true (it defaults to true alongside @@ -216,8 +217,8 @@ in # Private network namespace, attached to the hive bridge so the # runner reaches the forge via the gateway — and cannot reach # host-loopback (127.0.0.1:7000 dashboard, raw forge port, etc.). - # Requires `forge.behindGateway = true` (asserted in the options - # block above). See docs/networking/network.md. + # Requires `deploy.forgejo.behindGateway = true` (asserted in the + # options block above). See docs/networking/network.md. privateNetwork = true; hostBridge = networkCfg.bridgeName; diff --git a/nix/host-modules/hive-forge/default.nix b/nix/host-modules/hive-forge/default.nix index 43bc9ea1..b89b5120 100644 --- a/nix/host-modules/hive-forge/default.nix +++ b/nix/host-modules/hive-forge/default.nix @@ -75,7 +75,7 @@ let # links. Operators can still override via `cfg.rootUrl` for bespoke # shapes. defaultRootUrl = - if cfg.behindGateway then + if deployCfg.forgejo.behindGateway then let portSuffix = if gatewayCfg.httpsPort == 443 then "" else ":${toString gatewayCfg.httpsPort}"; in @@ -98,9 +98,9 @@ let # operator hasn't already declared that dest themselves (else CI-on + # an explicit `actions/checkout` entry would duplicate it). effectiveMirrors = - cfg.mirrors + deployCfg.forgejo.mirrors ++ lib.optional ( - ciEnabled && !(lib.any (m: m.dest == actionCheckoutMirror.dest) cfg.mirrors) + ciEnabled && !(lib.any (m: m.dest == actionCheckoutMirror.dest) deployCfg.forgejo.mirrors) ) actionCheckoutMirror; in { @@ -133,7 +133,7 @@ in Removed rather than defaulted to true so a config that turned it OFF fails here, where the line is, instead of silently gaining a login provider on the next rebuild. Drop the line; if it was - false, set services.hyperhive.swarm.forge.sso.clientSecretFile and + false, set services.hyperhive.deploy.forgejo.sso.clientSecretFile and services.hyperhive.swarm.authelia.url as the assertions describe. '') ]; @@ -178,7 +178,7 @@ in description = '' Public hostname for the forge. Doubles as both the forgejo `DOMAIN` setting (clone URLs forgejo advertises) AND the - gateway vhost server-name when `behindGateway = true` + gateway vhost server-name when `deploy.forgejo.behindGateway = true` (sub-domain routing — see `docs/networking/gateway.md`). Defaults to `forge.''${services.hyperhive.swarm.domain}` — the @@ -201,7 +201,10 @@ in publicUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; default = - if config.services.hyperhive.enable && cfg.behindGateway then "https://${cfg.domain}" else null; + if config.services.hyperhive.enable && deployCfg.forgejo.behindGateway then + "https://${cfg.domain}" + else + null; defaultText = lib.literalExpression '' if behindGateway then "https://''${domain}" else null ''; @@ -212,7 +215,7 @@ in the approval-queue's "review PR on forge" link) — sourced into every agent container + hive-c0re as `HIVE_FORGE_PUBLIC_URL`. - Defaults to `https://''${cfg.domain}` when `behindGateway = + Defaults to `https://''${cfg.domain}` when `deploy.forgejo.behindGateway = true` (the gateway vhost is genuinely reachable at that URL) and `null` otherwise. When `null`, the dashboard **hides** forge links rather than guessing one — see @@ -221,7 +224,7 @@ in container port is only an accident away from wrong on any deployment that isn't plain localhost). - **Set this explicitly if `behindGateway = false`** and the + **Set this explicitly if `deploy.forgejo.behindGateway = false`** and the forge is still reachable at a stable URL you want linked from the dashboard (e.g. `http://:''${toString cfg.httpPort}` for an all-LAN deployment) — leaving it unset there means the @@ -244,6 +247,60 @@ in ''; }; + rootUrl = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "https://forge.example.com/"; + description = '' + Override the auto-derived forgejo `ROOT_URL`. When `null` + (default), `ROOT_URL` is derived from `cfg.domain` + gateway + state, including the scheme: + + - `deploy.forgejo.behindGateway = true` → `https://''${cfg.domain}/`. The gateway + always terminates TLS (self-signed is the implicit floor when no + `gateway.tls.certDir` / ACME is set), so the forge is always + advertised over https. A non-canonical `gateway.httpsPort` is + appended as `:`. + - `deploy.forgejo.behindGateway = false` → `http://''${cfg.domain}:''${cfg.httpPort}/` + + The TLS scheme is derived automatically now, so you only need to + set this for a genuinely bespoke shape (e.g. an external reverse + proxy on a different host/path). Must end with `/` per forgejo's + `ROOT_URL` contract. + ''; + }; + + # The swarm's authelia is always registered as an OpenID Connect + # login source here — there is no toggle, for the same reason the + # forge itself has none. + # + # **Additive, never exclusive.** Forgejo keeps its local password + # database and gains an extra "sign in with" button; this does not + # disable local login. Deliberate: an identity provider that can take + # the forge offline when it hiccups is a worse forge than one with + # two ways in — which is also what makes always-on safe. + sso = { + clientId = lib.mkOption { + type = lib.types.str; + default = "forgejo"; + description = '' + OAuth2 client id this forge identifies itself with. Must match + the `id` of the corresponding entry in + `services.hyperhive.swarm.authelia.oidc.clients`. + ''; + }; + # The secret half is a path on the host that runs the forge, so it + # lives under `deploy.forgejo.sso` — see the block below. + }; + }; + + # What stays above is what the forge IS from any hive's point of view: its + # package, the names and ports it answers on, the URLs it advertises, and + # the client id it is registered under. What lives here is what the host + # running it decides — how it is served, what it mirrors, and where its + # host-local secrets sit. Same rule as ./swarm-victorialogs.nix, and the + # renames are in ./deploy.nix with the rest. + options.services.hyperhive.deploy.forgejo = { behindGateway = lib.mkOption { type = lib.types.bool; default = config.services.hyperhive.enable; @@ -274,29 +331,6 @@ in ''; }; - rootUrl = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "https://forge.example.com/"; - description = '' - Override the auto-derived forgejo `ROOT_URL`. When `null` - (default), `ROOT_URL` is derived from `cfg.domain` + gateway - state, including the scheme: - - - `behindGateway = true` → `https://''${cfg.domain}/`. The gateway - always terminates TLS (self-signed is the implicit floor when no - `gateway.tls.certDir` / ACME is set), so the forge is always - advertised over https. A non-canonical `gateway.httpsPort` is - appended as `:`. - - `behindGateway = false` → `http://''${cfg.domain}:''${cfg.httpPort}/` - - The TLS scheme is derived automatically now, so you only need to - set this for a genuinely bespoke shape (e.g. an external reverse - proxy on a different host/path). Must end with `/` per forgejo's - `ROOT_URL` contract. - ''; - }; - openFirewall = lib.mkOption { type = lib.types.bool; default = false; @@ -312,8 +346,8 @@ in **Breaking change**: this used to default to `true`. If you relied on the old default for external reach, add - `services.hyperhive.swarm.forge.openFirewall = true;` to your host - config before rebuilding. + `services.hyperhive.deploy.forgejo.openFirewall = true;` to your + host config before rebuilding. ''; }; @@ -360,45 +394,27 @@ in ''; }; - # The swarm's authelia is always registered as an OpenID Connect - # login source here — there is no toggle, for the same reason the - # forge itself has none. - # - # **Additive, never exclusive.** Forgejo keeps its local password - # database and gains an extra "sign in with" button; this does not - # disable local login. Deliberate: an identity provider that can take - # the forge offline when it hiccups is a worse forge than one with - # two ways in — which is also what makes always-on safe. - sso = { - clientId = lib.mkOption { - type = lib.types.str; - default = "forgejo"; - description = '' - OAuth2 client id this forge identifies itself with. Must match - the `id` of the corresponding entry in - `services.hyperhive.swarm.authelia.oidc.clients`. - ''; - }; + # The other half of the SSO pair: the client *id* is swarm-wide (it has + # to match authelia's register), the secret is a path on this host. The + # design note for the login source itself is with the id. + sso.clientSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/hyperhive/forge-oidc-secret"; + description = '' + Path **inside the forge container** holding the client + secret's plaintext. - clientSecretFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "/var/lib/hyperhive/forge-oidc-secret"; - description = '' - Path **inside the forge container** holding the client - secret's plaintext. + A path, never a value: an OIDC client secret has two holders + in two containers (authelia keeps a hash, this forge needs the + plaintext), and a literal written here would be rendered into + the world-readable nix store. - A path, never a value: an OIDC client secret has two holders - in two containers (authelia keeps a hash, this forge needs the - plaintext), and a literal written here would be rendered into - the world-readable nix store. - - Required when `enable` is set — deliberately no fallback. A - forge that boots with SSO half-configured presents as a login - button that always fails, which is harder to diagnose than an - eval error. - ''; - }; + Required when `enable` is set — deliberately no fallback. A + forge that boots with SSO half-configured presents as a login + button that always fails, which is harder to diagnose than an + eval error. + ''; }; hostSwarmControllerTokenFile = lib.mkOption { @@ -443,13 +459,13 @@ in # Both halves are gated on `behindGateway`: with it off the operator # fronts forgejo themselves, so this hive must neither claim the # vhost nor answer DNS for it. - services.hyperhive.gateway.localNames = lib.optional cfg.behindGateway cfg.domain; + services.hyperhive.gateway.localNames = lib.optional deployCfg.forgejo.behindGateway cfg.domain; # This swarm-ui quick-links entry, same `behindGateway` guard as the # vhost/DNS name above — with it off, this host doesn't actually # serve `cfg.domain`, so linking to it would be dead. See # `services.hyperhive.swarm.controller.links`'s description. - services.hyperhive.swarm.controller.links = lib.optional cfg.behindGateway { + services.hyperhive.swarm.controller.links = lib.optional deployCfg.forgejo.behindGateway { label = "Forge"; icon = "⚒"; url = "https://${cfg.domain}/"; @@ -466,15 +482,17 @@ in # what authelia compares against — this string agreeing with the # `location` block above it is the whole mechanism. A near miss is a # correctly minted token refused at the target. - services.hyperhive.swarm.otel.publishedScrapeTargets = lib.optionalAttrs cfg.behindGateway { - forgejo = "https://${cfg.domain}/metrics"; - }; + services.hyperhive.swarm.otel.publishedScrapeTargets = + lib.optionalAttrs deployCfg.forgejo.behindGateway + { + forgejo = "https://${cfg.domain}/metrics"; + }; # `server_name = forge.domain`, proxies all `/` → forgejo. Tuned for # git: `client_max_body_size 1G`, `proxy_read_timeout 1h` (multi-GB # clones). SSH stays direct on `forge.sshPort`. See # `docs/networking/gateway.md`. - services.nginx.virtualHosts = lib.optionalAttrs cfg.behindGateway { + services.nginx.virtualHosts = lib.optionalAttrs deployCfg.forgejo.behindGateway { "${cfg.domain}" = (gatewayCfg.lib.tlsFor cfg.domain) // { listen = gatewayCfg.lib.listen; extraConfig = gatewayCfg.lib.securityHeaders; @@ -543,10 +561,10 @@ in { # Fail at EVAL, not at boot. The alternative failure is a login # button that always 401s, three layers from the missing file. - assertion = cfg.sso.clientSecretFile != null; + assertion = deployCfg.forgejo.sso.clientSecretFile != null; message = '' The forge's SSO login source requires - services.hyperhive.swarm.forge.sso.clientSecretFile — the path + services.hyperhive.deploy.forgejo.sso.clientSecretFile — the path (inside the forge container) holding the OIDC client secret's plaintext. @@ -600,7 +618,7 @@ in # splits on the single slash to create the org + repo. assertion = lib.all (m: lib.length (lib.splitString "/" m.dest) == 2) effectiveMirrors; message = '' - Every services.hyperhive.swarm.forge.mirrors[].dest must be exactly + Every services.hyperhive.deploy.forgejo.mirrors[].dest must be exactly "/" (one slash). Got: ${lib.concatMapStringsSep ", " (m: m.dest) effectiveMirrors} ''; } @@ -618,7 +636,7 @@ in ]) ) effectiveMirrors; message = '' - services.hyperhive.swarm.forge.mirrors[].dest must not place a mirror + services.hyperhive.deploy.forgejo.mirrors[].dest must not place a mirror in a hive-c0re-managed org (config / shared / agents / core) — those are provisioned by hive-c0re and a mirror there would collide. Use a dedicated org (e.g. "actions/checkout"). @@ -771,7 +789,7 @@ in # scraper at the gateway, so a second credential system per # service would buy nothing and would be the one that stops # getting rotated. - metrics.ENABLED = cfg.behindGateway; + metrics.ENABLED = deployCfg.forgejo.behindGateway; # The two per-dimension breakdowns, on the same condition as # the endpoint itself: `gitea_issues_by_label{label=…}` and # `gitea_issues_by_repository{repository=…}`. Off by default @@ -790,8 +808,8 @@ in # forge that grew to thousands of repos would want this # revisited — that is a real trigger, unlike a time-based one: # `count(gitea_issues_by_repository)` answers it directly. - metrics.ENABLED_ISSUE_BY_LABEL = cfg.behindGateway; - metrics.ENABLED_ISSUE_BY_REPOSITORY = cfg.behindGateway; + metrics.ENABLED_ISSUE_BY_LABEL = deployCfg.forgejo.behindGateway; + metrics.ENABLED_ISSUE_BY_REPOSITORY = deployCfg.forgejo.behindGateway; # Repo migrations / pull-mirrors fetch from the source # URL *inside* Forgejo. hyperhive code is synced from # `localhost` (and the host LAN), which Forgejo's @@ -1025,9 +1043,9 @@ in script = '' set -euo pipefail - secret=$(cat ${lib.escapeShellArg cfg.sso.clientSecretFile}) + secret=$(cat ${lib.escapeShellArg deployCfg.forgejo.sso.clientSecretFile}) if [ -z "$secret" ]; then - echo "empty OIDC client secret at ${cfg.sso.clientSecretFile}" >&2 + echo "empty OIDC client secret at ${deployCfg.forgejo.sso.clientSecretFile}" >&2 exit 1 fi @@ -1204,7 +1222,7 @@ in # Same case, same reasoning: this host minted the secret, so it can # say where the forge will find it. - services.hyperhive.swarm.forge.sso.clientSecretFile = lib.mkIf ssoLocal ( + services.hyperhive.deploy.forgejo.sso.clientSecretFile = lib.mkIf ssoLocal ( lib.mkDefault forgeSecretPath ); @@ -1298,7 +1316,7 @@ in set -euo pipefail src=${lib.escapeShellArg "/var/lib/nixos-containers/hive-forge${swarmControllerTokenPath}"} - dst=${lib.escapeShellArg cfg.hostSwarmControllerTokenFile} + dst=${lib.escapeShellArg deployCfg.forgejo.hostSwarmControllerTokenFile} for _ in $(seq 1 60); do [ -s "$src" ] && break @@ -1313,7 +1331,7 @@ in ''; }; - networking.firewall = lib.mkIf cfg.openFirewall { + networking.firewall = lib.mkIf deployCfg.forgejo.openFirewall { allowedTCPPorts = [ cfg.httpPort cfg.sshPort diff --git a/nix/host-modules/swarm-authelia.nix b/nix/host-modules/swarm-authelia.nix index 9684293f..afb989f6 100644 --- a/nix/host-modules/swarm-authelia.nix +++ b/nix/host-modules/swarm-authelia.nix @@ -1361,7 +1361,7 @@ in # Denied or client-scoped depending on whether a # collector is registered — see `metricsRule` above, # which is where the reasoning for both halves lives. - lib.optional forgeCfg.behindGateway metricsRule + lib.optional deployCfg.forgejo.behindGateway metricsRule # Also guarded on the domain being set: without it a null # apex would render a rule matching the string "null". ++ lib.optional (deployCfg.swarm-ui.enable && swarmDomain != null) { diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index e46f2c2e..bb7d8ddd 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -413,8 +413,8 @@ in forgeTokenFile = lib.mkOption { type = lib.types.nullOr lib.types.str; - # `services.hyperhive.swarm.forge` has no `enable` of its own to - # check — the module activates on the general + # The forge has no `enable` of its own to check — neither half of + # its split namespace carries one — the module activates on the general # `config.services.hyperhive.enable` instead (see # `hive-forge/default.nix`'s own `config = lib.mkIf # config.services.hyperhive.enable { ... }`), so that's the @@ -423,7 +423,8 @@ in # that turns swarm-controller on — caught in review, not by # `nix flake check` (nothing in its checked combinations forced # this particular default to actually evaluate). - default = if config.services.hyperhive.enable then forgeCfg.hostSwarmControllerTokenFile else null; + default = + if config.services.hyperhive.enable then deployCfg.forgejo.hostSwarmControllerTokenFile else null; defaultText = lib.literalExpression '' forge's own `hostSwarmControllerTokenFile` when this host runs hyperhive at all (forge has no separate enable), else null @@ -440,7 +441,7 @@ in `hive-forge/default.nix`, it has no `enable` of its own). Override explicitly if forge's actual token file ends up somewhere else — copy it out of forge's - {option}`services.hyperhive.swarm.forge.hostSwarmControllerTokenFile` + {option}`services.hyperhive.deploy.forgejo.hostSwarmControllerTokenFile` with whatever secret management this deployment already uses, the same shape `swarm.nix`'s `clientSecretFile` documents for its own cross-host case. `null` means no forge access — the diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 396d356a..9af2b083 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -72,6 +72,23 @@ let swarm.wireguard.privateKeyFile = "/etc/wireguard/hive.key"; }; + # Same shape for the forge, which SPLIT rather than moving whole: the five + # host-side options are set here through their pre-rename paths while the + # rest of `swarm.forge` stays put. All five are defined so that dropping any + # single shim entry fails the eval, not just the two the assertion reads. + forgeOldPath = hive { + swarm.forge.behindGateway = true; + swarm.forge.openFirewall = true; + swarm.forge.hostSwarmControllerTokenFile = "/etc/forge/sc.token"; + swarm.forge.sso.clientSecretFile = "/etc/forge/oidc.secret"; + swarm.forge.mirrors = [ + { + upstream = "https://example.invalid/tool"; + dest = "mirrors/tool"; + } + ]; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -205,6 +222,20 @@ let in wg.ips == [ "10.100.0.1/24" ] && wg.privateKeyFile == "/etc/wireguard/hive.key"; } + { + # Same reasoning one namespace over, plus the shape the mesh did not + # have: `mirrors` is a single option of a list-of-submodule type, so its + # one rename entry has to carry a whole compound value rather than a + # scalar. Both arms read a rendered effect — the host firewall and the + # env var c0re seeds mirrors from — not the option. + name = "a config written against the pre-rename forge paths still opens the firewall and seeds the mirror"; + ok = + let + ports = forgeOldPath.networking.firewall.allowedTCPPorts; + seeded = builtins.fromJSON forgeOldPath.systemd.services.hive-c0re.environment.HYPERHIVE_FORGE_MIRRORS; + in + builtins.elem 3000 ports && builtins.any (m: m.dest == "mirrors/tool") seeded; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA From 7003d14d2c903525f276b6b7c640bbc26c17f4c7 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 11:23:47 +0200 Subject: [PATCH 3/6] deploy: split the homeserver's host decisions out of swarm.matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swarm.*` is what a hive needs to be a *client* of the swarm. For the homeserver that is what it IS from anywhere: its package, the name it answers to, the ports and URLs it is reached on, and the client id it is registered under. Whether it is exposed, which peers it trusts, how large a request it accepts and where its host-local secrets sit are decisions of the machine running it, so openFirewall, trustedServers, maxRequestSize, registrationTokenFile, gui.enable and sso.clientSecretFile move to `deploy.matrix.*`. Two sub-blocks split rather than moving whole, on their own evidence. `gui.enable` is whether THIS host serves the web client; `gui.package` is which client, an artifact identity, and stays. `sso.clientSecretFile` is a path on one host; `clientId` must match the id in authelia's register, so it is swarm-wide. Each half now points at the other, because the rendered docs put them on separate pages. hive-gateway passed the whole `swarm.matrix` attrset into vhosts.nix, so that file read a moving option through an argument with no option path anywhere in it. It now takes `matrixDeployCfg` beside `matrixCfg` — the only shape that carries a split namespace across that boundary. While there: vhosts.nix read `matrixCfg.enable`, which has been a rename alias for `deploy.matrix.enable` since the enable moved. Reading it made the module system print `Obsolete option services.hyperhive.swarm.matrix. enable is used` on EVERY evaluation of every host — a deprecation warning no operator could silence, because the config tripping it was ours. That shim lives in hive-matrix.nix rather than in this file's table, which is why deploy.nix's header claim to be their single home is now qualified in the new block's comment. glue-matrix-bao-token.nix read the registration token through its own `matrixCfg` alias; with that read repointed, the binding had no reader left, so it goes, and the comment naming it is reworded. module-eval gains a case configuring a hive through all six OLD paths and asserting two rendered effects — the host firewall's port list and the container's bind-mount table — because the new paths evaluate fine without the shims. `gui.enable` is set to the opposite of its default so the definition has to land rather than agreeing with it by accident. --- docs/tools/hivectl.md | 2 +- docs/web-ui/dashboard.md | 2 +- hive-c0re/src/dashboard/state_snapshot.rs | 2 +- hivectl/src/open.rs | 2 +- nix/host-modules/deploy.nix | 31 +++ nix/host-modules/glue-matrix-bao-token.nix | 10 +- nix/host-modules/hive-c0re/environment.nix | 4 +- nix/host-modules/hive-gateway/default.nix | 2 + nix/host-modules/hive-gateway/vhosts.nix | 8 +- nix/host-modules/hive-matrix.nix | 250 +++++++++++---------- nix/module-eval.nix | 30 +++ 11 files changed, 212 insertions(+), 131 deletions(-) diff --git a/docs/tools/hivectl.md b/docs/tools/hivectl.md index d985d582..9c9fb8df 100644 --- a/docs/tools/hivectl.md +++ b/docs/tools/hivectl.md @@ -315,5 +315,5 @@ an error. A surface has no URL when it isn't browser-reachable: `home` needs `services.hyperhive.domain`; `forge` needs `services.hyperhive.deploy.forgejo.behindGateway = true`; `matrix` needs -`services.hyperhive.swarm.matrix.gui.enable = true`. In those cases the command +`services.hyperhive.deploy.matrix.gui.enable = true`. In those cases the command exits with a hint naming the option to set. diff --git a/docs/web-ui/dashboard.md b/docs/web-ui/dashboard.md index 116ae44a..77fc50a2 100644 --- a/docs/web-ui/dashboard.md +++ b/docs/web-ui/dashboard.md @@ -627,7 +627,7 @@ A static matrix web client (default `pkgs.fluffychat-web` rebuilt with `--base-href /matrix/`, swappable via `services.hyperhive.swarm.matrix.gui.package`) served by the hive-gateway nginx container at `/matrix/` when -`services.hyperhive.swarm.matrix.gui.enable` is on (defaults to +`services.hyperhive.deploy.matrix.gui.enable` is on (defaults to `matrix.enable`). c0re signals availability via the `HIVE_MATRIX_GUI_ENABLED` env var → `state.matrix_gui_enabled` in `/api/state`; the gateway does the actual static serving. diff --git a/hive-c0re/src/dashboard/state_snapshot.rs b/hive-c0re/src/dashboard/state_snapshot.rs index be12d782..32556ae3 100644 --- a/hive-c0re/src/dashboard/state_snapshot.rs +++ b/hive-c0re/src/dashboard/state_snapshot.rs @@ -75,7 +75,7 @@ pub(super) struct StateSnapshot { forge_present: bool, /// Whether the matrix GUI is reachable at `/matrix/`. Sourced from /// `HIVE_MATRIX_GUI_ENABLED` env var (set by the c0re NixOS module - /// when `services.hyperhive.swarm.matrix.gui.enable` is on). The gateway + /// when `services.hyperhive.deploy.matrix.gui.enable` is on). The gateway /// (`nix/host-modules/hive-gateway/vhosts.nix`) does the actual `/matrix/` static serving; /// this flag is just an availability signal for iris's dashboard /// chrome so the `M4TR1X →` tab doesn't flash when the GUI is off. diff --git a/hivectl/src/open.rs b/hivectl/src/open.rs index 16e366dd..4ebf7a56 100644 --- a/hivectl/src/open.rs +++ b/hivectl/src/open.rs @@ -30,7 +30,7 @@ pub(crate) async fn open_url(socket: &Path, target: OpenTarget) -> Result<()> { ), OpenTarget::Matrix => ( urls.matrix, - "the matrix GUI URL needs `services.hyperhive.swarm.matrix.gui.enable = true`", + "the matrix GUI URL needs `services.hyperhive.deploy.matrix.gui.enable = true`", ), }; let url = url.with_context(|| format!("no URL available for this surface — {hint}"))?; diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 78d74f71..2ad3738f 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -130,6 +130,37 @@ in [ "services" "hyperhive" "deploy" "forgejo" "hostSwarmControllerTokenFile" ] ) + # The homeserver, split the same way. `swarm.matrix` keeps what it IS to + # every hive — its package, the name it answers to, the ports and URLs it + # is reached on, the client id it is registered under. These six are what + # the host running it decides. ⚠️ `enable` is NOT here: it was renamed + # earlier and its shim lives in ./hive-matrix.nix, so that file is the one + # to check before assuming this table is the whole matrix story. + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "openFirewall" ] + [ "services" "hyperhive" "deploy" "matrix" "openFirewall" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "trustedServers" ] + [ "services" "hyperhive" "deploy" "matrix" "trustedServers" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "maxRequestSize" ] + [ "services" "hyperhive" "deploy" "matrix" "maxRequestSize" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "registrationTokenFile" ] + [ "services" "hyperhive" "deploy" "matrix" "registrationTokenFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "gui" "enable" ] + [ "services" "hyperhive" "deploy" "matrix" "gui" "enable" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "matrix" "sso" "clientSecretFile" ] + [ "services" "hyperhive" "deploy" "matrix" "sso" "clientSecretFile" ] + ) + # Retention is read only where the container is defined, so it is a # decision of the host running the store rather than something the swarm # agrees on. The two stores keep everything else — package, domain, port diff --git a/nix/host-modules/glue-matrix-bao-token.nix b/nix/host-modules/glue-matrix-bao-token.nix index fabdba5a..89f77cc2 100644 --- a/nix/host-modules/glue-matrix-bao-token.nix +++ b/nix/host-modules/glue-matrix-bao-token.nix @@ -31,7 +31,6 @@ let hyperhiveCfg = config.services.hyperhive; deployCfg = hyperhiveCfg.deploy; baoCfg = hyperhiveCfg.swarm.bao; - matrixCfg = hyperhiveCfg.swarm.matrix; baoDeploy = deployCfg.bao; @@ -48,8 +47,9 @@ let # A literal, not an option — ./hive-matrix.nix names its container # `containers.hive-matrix` directly and declares no `machine` to derive it # from, which the trust-bundle call in that file already says out loud. - # ⚠️ `matrixCfg.machine` parses fine and fails at module-system resolution, - # so this is the kind of mistake only reading the target module catches. + # ⚠️ A `swarm.matrix.machine` read parses fine and fails at module-system + # resolution, so this is the kind of mistake only reading the target module + # catches. matrixMachine = "hive-matrix"; in { @@ -115,8 +115,8 @@ in fi umask 077 - printf '%s\n' "$token" > ${lib.escapeShellArg (toString matrixCfg.registrationTokenFile)} - chmod 0600 ${lib.escapeShellArg (toString matrixCfg.registrationTokenFile)} + printf '%s\n' "$token" > ${lib.escapeShellArg (toString deployCfg.matrix.registrationTokenFile)} + chmod 0600 ${lib.escapeShellArg (toString deployCfg.matrix.registrationTokenFile)} ''; }; }; diff --git a/nix/host-modules/hive-c0re/environment.nix b/nix/host-modules/hive-c0re/environment.nix index 9882bead..5a56cf7b 100644 --- a/nix/host-modules/hive-c0re/environment.nix +++ b/nix/host-modules/hive-c0re/environment.nix @@ -200,7 +200,7 @@ in # one. HIVE_MATRIX_API_URL = config.services.hyperhive.swarm.matrix.apiUrl; } -// lib.optionalAttrs config.services.hyperhive.swarm.matrix.gui.enable { +// lib.optionalAttrs config.services.hyperhive.deploy.matrix.gui.enable { # Availability flags read by the dashboard's `/api/state`. # Matrix GUI lives entirely on the gateway nginx (matrix tab # only shows when both flags are on). Gateway routing detail: @@ -229,7 +229,7 @@ in // lib.optionalAttrs ( - config.services.hyperhive.swarm.matrix.gui.enable + config.services.hyperhive.deploy.matrix.gui.enable && config.services.hyperhive.swarm.matrix.gatewayHost != null ) { diff --git a/nix/host-modules/hive-gateway/default.nix b/nix/host-modules/hive-gateway/default.nix index c3130905..fe71e7f7 100644 --- a/nix/host-modules/hive-gateway/default.nix +++ b/nix/host-modules/hive-gateway/default.nix @@ -22,6 +22,7 @@ let # same list rather than each deciding what "a swarm service" means. swarmServiceDomains = config.services.hyperhive.swarm.serviceDomains; matrixCfg = config.services.hyperhive.swarm.matrix; + matrixDeployCfg = config.services.hyperhive.deploy.matrix; networkCfg = config.services.hyperhive.network; # Every vhost claiming `default_server`, ours and the operator's @@ -104,6 +105,7 @@ let cfg errorPages matrixCfg + matrixDeployCfg hyperhiveDomain dashboardDist swaggerUiTheme diff --git a/nix/host-modules/hive-gateway/vhosts.nix b/nix/host-modules/hive-gateway/vhosts.nix index 0a9d2752..7c4b1ce9 100644 --- a/nix/host-modules/hive-gateway/vhosts.nix +++ b/nix/host-modules/hive-gateway/vhosts.nix @@ -15,7 +15,8 @@ { lib, cfg, # services.hyperhive.gateway - matrixCfg, + matrixCfg, # services.hyperhive.swarm.matrix — what the homeserver IS + matrixDeployCfg, # services.hyperhive.deploy.matrix — what THIS host runs hyperhiveDomain, dashboardDist, swaggerUiTheme, # nix/packages/swagger-ui-theme.nix: has index.html + hyperhive-theme.css @@ -44,7 +45,8 @@ let # `/matrix/*` → 301 → `matrix./$1` (legacy deep-link # shim during the fluffychat sub-domain move). See `docs/networking/gateway.md`. matrixRedirectLocations = - lib.optionalAttrs (matrixCfg.enable && matrixCfg.gui.enable && matrixCfg.gatewayHost != null) + lib.optionalAttrs + (matrixDeployCfg.enable && matrixDeployCfg.gui.enable && matrixCfg.gatewayHost != null) ( let target = "${publicScheme}://${matrixCfg.gatewayHost}${publicPortSuffix}"; @@ -62,7 +64,7 @@ let # clients at `matrixCfg.gatewayHost` when set; falls back to direct # `:`. CORS `*` per matrix spec. The `m.server` # port-8448 carve-out is documented inline. See `docs/networking/gateway.md`. - wellKnownLocations = lib.optionalAttrs matrixCfg.enable ( + wellKnownLocations = lib.optionalAttrs matrixDeployCfg.enable ( let clientBaseUrl = if matrixCfg.gatewayHost != null then diff --git a/nix/host-modules/hive-matrix.nix b/nix/host-modules/hive-matrix.nix index cdeaf9ef..fe6e39e0 100644 --- a/nix/host-modules/hive-matrix.nix +++ b/nix/host-modules/hive-matrix.nix @@ -178,7 +178,7 @@ in Removed rather than defaulted to true so a config that turned it OFF fails here, where the line is, instead of silently gaining a login flow on the next rebuild. Drop the line; if it was false, - set services.hyperhive.swarm.matrix.sso.clientSecretFile and + set services.hyperhive.deploy.matrix.sso.clientSecretFile and services.hyperhive.swarm.authelia.url as the assertions describe. '') ]; @@ -312,6 +312,91 @@ in ''; }; + allowEncryption = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Server-side switch for matrix end-to-end encryption — sets + tuwunel's `allow_encryption`. Off by default: on the hive-internal + homeserver the operator already controls the transport, so server + E2EE adds key-management overhead (cross-signing, device + verification, undecryptable-message recovery) without a clear + threat-model win for the common single-hive case. Turn on when + agents join encrypted rooms on external / federated homeservers, + or when the operator wants message contents opaque to the + homeserver admin. Independent of the agent matrix client, which + always supports decryption so it can read encrypted rooms it is + invited to regardless of this flag; this option only governs + whether THIS homeserver permits room encryption. + ''; + }; + + gui = { + # Whether this host serves it is `deploy.matrix.gui.enable`. + package = lib.mkOption { + type = lib.types.package; + default = fluffychat-web-fixed; + defaultText = lib.literalMD '' + `pkgs.fluffychat-web` with a `postInstall` patch that adds + the three files `flutter341.buildFlutterApplication` skips. + ''; + description = '' + Static web client dist served at `matrix./`. Override + to swap fluffychat for hydrogen-web, cinny, element-web, or + an out-of-tree dist — any replacement is mounted at the + sub-domain root with the upstream-default ``, + no sub-path gymnastics needed. + ''; + }; + }; + + # This homeserver always delegates login to the swarm's authelia, as + # an OIDC relying party — matrix SSO (`m.login.sso`), offered + # alongside password login. No toggle: a homeserver in a swarm is a + # client of that swarm's identity provider. + # + # ⚠️ Not to be confused with tuwunel's `oidc_*` settings, which point + # the other way: those make this homeserver an *authorization server* + # for matrix clients. This family makes it a *client* of an external + # identity provider. The two share the protocol's name and answer + # opposite questions. + # + # This **adds** a way in. Password login keeps working: an identity + # provider that can take the homeserver offline when it hiccups is a + # worse homeserver than one with two ways in — which is also what + # makes always-on safe. Making authelia the *only* path is a + # separate, reversible switch (tuwunel's `login_with_password`), + # deliberately not folded in here. + # + # ⚠️ Matrix SSO lives **inside** the homeserver, never behind a + # forward-auth proxy: the client-server API is spoken by non-browser + # clients holding matrix access tokens — every agent's own + # `hive-matrix-daemon` — plus federation, and a proxy in front of + # `/_matrix/` breaks all of it. + sso = { + clientId = lib.mkOption { + type = lib.types.str; + default = "tuwunel"; + description = '' + OAuth2 client id this homeserver identifies itself with. Must + match the `id` of the corresponding entry in + `services.hyperhive.swarm.authelia.oidc.clients`. + + The secret it pairs with is a host path, so it lives at + `services.hyperhive.deploy.matrix.sso.clientSecretFile`. + ''; + }; + }; + }; + + # What stays above is what the homeserver IS from any hive's point of view: + # its package, the name it answers to, the ports and URLs it is reached on, + # and the client id it is registered under. What lives here is what the host + # running it decides — whether it is exposed, which peers it trusts, how large + # a request it accepts, and where its host-local secrets sit. Same rule as + # ./swarm-victorialogs.nix; `enable` already lives in ./deploy.nix, which also + # carries the renames. + options.services.hyperhive.deploy.matrix = { openFirewall = lib.mkOption { type = lib.types.bool; default = false; @@ -327,7 +412,7 @@ in **Breaking change**: this used to default to `true`. If you relied on the old default for external reach, add - `services.hyperhive.swarm.matrix.openFirewall = true;` to your host + `services.hyperhive.deploy.matrix.openFirewall = true;` to your host config before rebuilding. Note: federation (the matrix-spec well-known port 8448) is @@ -381,112 +466,43 @@ in ''; }; - allowEncryption = lib.mkOption { + gui.enable = lib.mkOption { type = lib.types.bool; - default = false; + default = deployCfg.matrix.enable; + defaultText = lib.literalExpression "config.services.hyperhive.deploy.matrix.enable"; description = '' - Server-side switch for matrix end-to-end encryption — sets - tuwunel's `allow_encryption`. Off by default: on the hive-internal - homeserver the operator already controls the transport, so server - E2EE adds key-management overhead (cross-signing, device - verification, undecryptable-message recovery) without a clear - threat-model win for the common single-hive case. Turn on when - agents join encrypted rooms on external / federated homeservers, - or when the operator wants message contents opaque to the - homeserver admin. Independent of the agent matrix client, which - always supports decryption so it can read encrypted rooms it is - invited to regardless of this flag; this option only governs - whether THIS homeserver permits room encryption. + Serve a matrix web client at `matrix.''${services.hyperhive.domain}/`. + Requires `swarm.matrix.gatewayHost != null` (default `matrix.` + when hive-domain set); the gateway itself always runs. When + off, the dashboard's `M4TR1X →` tab is hidden. See + `docs/networking/gateway.md` for the discovery flow that lets clients + auto-find the sub-domain. The client build itself is + `swarm.matrix.gui.package` — which client, as opposed to whether + this host serves it. ''; }; - gui = { - enable = lib.mkOption { - type = lib.types.bool; - default = deployCfg.matrix.enable; - defaultText = lib.literalExpression "config.services.hyperhive.deploy.matrix.enable"; - description = '' - Serve a matrix web client at `matrix.''${services.hyperhive.domain}/`. - Requires `matrix.gatewayHost != null` (default `matrix.` - when hive-domain set); the gateway itself always runs. When - off, the dashboard's `M4TR1X →` tab is hidden. See - `docs/networking/gateway.md` for the discovery flow that lets clients - auto-find the sub-domain. - ''; - }; + sso.clientSecretFile = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "/var/lib/tuwunel-oidc/tuwunel.secret"; + description = '' + Path **inside the matrix container** holding the client + secret's plaintext. - package = lib.mkOption { - type = lib.types.package; - default = fluffychat-web-fixed; - defaultText = lib.literalMD '' - `pkgs.fluffychat-web` with a `postInstall` patch that adds - the three files `flutter341.buildFlutterApplication` skips. - ''; - description = '' - Static web client dist served at `matrix./`. Override - to swap fluffychat for hydrogen-web, cinny, element-web, or - an out-of-tree dist — any replacement is mounted at the - sub-domain root with the upstream-default ``, - no sub-path gymnastics needed. - ''; - }; - }; + A path, never a value: an OIDC client secret has two holders + in two containers (authelia keeps a hash, this homeserver + needs the plaintext), and a literal written here would be + rendered into the world-readable nix store. - # This homeserver always delegates login to the swarm's authelia, as - # an OIDC relying party — matrix SSO (`m.login.sso`), offered - # alongside password login. No toggle: a homeserver in a swarm is a - # client of that swarm's identity provider. - # - # ⚠️ Not to be confused with tuwunel's `oidc_*` settings, which point - # the other way: those make this homeserver an *authorization server* - # for matrix clients. This family makes it a *client* of an external - # identity provider. The two share the protocol's name and answer - # opposite questions. - # - # This **adds** a way in. Password login keeps working: an identity - # provider that can take the homeserver offline when it hiccups is a - # worse homeserver than one with two ways in — which is also what - # makes always-on safe. Making authelia the *only* path is a - # separate, reversible switch (tuwunel's `login_with_password`), - # deliberately not folded in here. - # - # ⚠️ Matrix SSO lives **inside** the homeserver, never behind a - # forward-auth proxy: the client-server API is spoken by non-browser - # clients holding matrix access tokens — every agent's own - # `hive-matrix-daemon` — plus federation, and a proxy in front of - # `/_matrix/` breaks all of it. - sso = { - clientId = lib.mkOption { - type = lib.types.str; - default = "tuwunel"; - description = '' - OAuth2 client id this homeserver identifies itself with. Must - match the `id` of the corresponding entry in - `services.hyperhive.swarm.authelia.oidc.clients`. - ''; - }; - - clientSecretFile = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - example = "/var/lib/tuwunel-oidc/tuwunel.secret"; - description = '' - Path **inside the matrix container** holding the client - secret's plaintext. - - A path, never a value: an OIDC client secret has two holders - in two containers (authelia keeps a hash, this homeserver - needs the plaintext), and a literal written here would be - rendered into the world-readable nix store. - - Required when `enable` is set — deliberately no fallback. A - homeserver that boots with SSO half-configured is worse than - one that fails to evaluate: tuwunel reads OIDC from its - config file rather than a database row, so a malformed block - can stop the server outright instead of merely hiding a - button. - ''; - }; + Deliberately no fallback — a homeserver that boots with SSO + half-configured is worse than one that fails to evaluate: + tuwunel reads OIDC from its config file rather than a database + row, so a malformed block can stop the server outright instead + of merely hiding a button. The id this pairs with is + `swarm.matrix.sso.clientId`, which is swarm-wide because it must + match authelia's register. + ''; }; }; @@ -520,7 +536,7 @@ in # page). See `services.hyperhive.swarm.controller.links`'s # description. services.hyperhive.swarm.controller.links = - lib.optional (cfg.gatewayHost != null && cfg.gui.enable) + lib.optional (cfg.gatewayHost != null && deployCfg.matrix.gui.enable) { label = "Matrix"; icon = "💬"; @@ -534,7 +550,7 @@ in # contributes instead of replacing it. # # The dashboard needs no equivalent — it routes by path. - services.nginx.appendHttpConfig = lib.optionalString cfg.gui.enable '' + services.nginx.appendHttpConfig = lib.optionalString deployCfg.matrix.gui.enable '' map $http_accept $matrix_spa_target { default "/__matrix_spa_no_html_fallback"; "~*text/html" "/index.html"; @@ -569,7 +585,7 @@ in ''; }; } - // lib.optionalAttrs cfg.gui.enable { + // lib.optionalAttrs deployCfg.matrix.gui.enable { # fluffychat at sub-domain root, SPA-fallback via the # Accept-header `$matrix_spa_target` map above. "/" = { @@ -588,7 +604,7 @@ in ''; }; } - // lib.optionalAttrs (!cfg.gui.enable) { + // lib.optionalAttrs (!deployCfg.matrix.gui.enable) { "/" = { return = "404"; }; @@ -618,10 +634,10 @@ in # Fail at EVAL, not at boot. tuwunel reads its identity providers # from the config file, so a half-configured one does not hide a # login button — it can stop the homeserver from starting at all. - assertion = cfg.sso.clientSecretFile != null; + assertion = deployCfg.matrix.sso.clientSecretFile != null; message = '' This homeserver's SSO login flow requires - services.hyperhive.swarm.matrix.sso.clientSecretFile — the path + services.hyperhive.deploy.matrix.sso.clientSecretFile — the path (inside the matrix container) holding the OIDC client secret's plaintext. @@ -683,7 +699,7 @@ in # Same case, same reasoning: this host minted the secret, so it can say # where the homeserver will find it. - services.hyperhive.swarm.matrix.sso.clientSecretFile = lib.mkIf ssoLocal ( + services.hyperhive.deploy.matrix.sso.clientSecretFile = lib.mkIf ssoLocal ( lib.mkDefault matrixSecretPath ); @@ -729,7 +745,7 @@ in set -euo pipefail src=${lib.escapeShellArg "${autheliaCfg.hostClientSecretDir}/${cfg.sso.clientId}.secret"} - dst=${lib.escapeShellArg "/var/lib/nixos-containers/hive-matrix${toString cfg.sso.clientSecretFile}"} + dst=${lib.escapeShellArg "/var/lib/nixos-containers/hive-matrix${toString deployCfg.matrix.sso.clientSecretFile}"} # authelia's container is up, but its first-boot generator may # still be minting. Bounded wait, then fail: a silent skip here @@ -811,7 +827,7 @@ in ); system.activationScripts.hive-matrix-register-token = lib.stringAfter [ "var" ] '' - tokenFile=${lib.escapeShellArg (toString cfg.registrationTokenFile)} + tokenFile=${lib.escapeShellArg (toString deployCfg.matrix.registrationTokenFile)} if [ ! -s "$tokenFile" ]; then mkdir -p "$(dirname "$tokenFile")" head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' > "$tokenFile" @@ -833,8 +849,8 @@ in # Read-only bind of the host-managed registration token; tuwunel # reads it via systemd LoadCredential below (not directly). bindMounts = { - ${cfg.registrationTokenFile} = { - hostPath = cfg.registrationTokenFile; + ${deployCfg.matrix.registrationTokenFile} = { + hostPath = deployCfg.matrix.registrationTokenFile; isReadOnly = true; }; } @@ -923,11 +939,11 @@ in # `address` + `port` are upstream `listOf` — wrap singles. address = [ "0.0.0.0" ]; port = [ cfg.httpPort ]; - max_request_size = cfg.maxRequestSize; + max_request_size = deployCfg.matrix.maxRequestSize; # Federation enabled at the protocol level; empty # trustedServers keeps it effectively closed. allow_federation = true; - trusted_servers = cfg.trustedServers; + trusted_servers = deployCfg.matrix.trustedServers; # Token-gated registration. The absent # `yes_i_am_very_very_sure_…_open_registration_…` flag # keeps the server closed to anyone without the token. @@ -995,18 +1011,18 @@ in # host-side chown :tuwunel / GID-pin gymnastics needed. # See `man systemd.exec` → LoadCredential. systemd.services.tuwunel.serviceConfig.LoadCredential = [ - "registration_token:${toString cfg.registrationTokenFile}" + "registration_token:${toString deployCfg.matrix.registrationTokenFile}" # Same mechanism, second secret. tuwunel re-reads this file on # every OAuth exchange, not just at startup, so it has to # outlive the unit's start — a credentials path does. - "oidc_client_secret:${toString cfg.sso.clientSecretFile}" + "oidc_client_secret:${toString deployCfg.matrix.sso.clientSecretFile}" ]; environment.systemPackages = [ cfg.package ]; }; }; - networking.firewall = lib.mkIf cfg.openFirewall { + networking.firewall = lib.mkIf deployCfg.matrix.openFirewall { allowedTCPPorts = [ cfg.httpPort ]; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 9af2b083..301ce64c 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -89,6 +89,21 @@ let ]; }; + # The homeserver's turn to split. All six host-side options are set through + # their pre-rename paths — including `gui.enable`, whose value is deliberately + # the opposite of its default so the definition has to actually land. All six + # so that dropping any single shim entry fails the eval, not just the two the + # assertion reads. + matrixOldPath = hive { + deploy.matrix.enable = true; + swarm.matrix.openFirewall = true; + swarm.matrix.trustedServers = [ "matrix.example.invalid" ]; + swarm.matrix.maxRequestSize = 31457280; + swarm.matrix.registrationTokenFile = "/etc/matrix/register.token"; + swarm.matrix.gui.enable = false; + swarm.matrix.sso.clientSecretFile = "/etc/matrix/oidc.secret"; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -236,6 +251,21 @@ let in builtins.elem 3000 ports && builtins.any (m: m.dest == "mirrors/tool") seeded; } + { + # Third split, and the one whose readers were hardest to see: the + # registration token is read only through a `let` alias in another + # module, so no full path names it anywhere. Both arms read a rendered + # effect — the host firewall and the container's bind-mount table — so a + # rename that resolves but stops reaching the module still fails. + name = "a config written against the pre-rename matrix paths still opens the port and mounts the token"; + ok = + let + ports = matrixOldPath.networking.firewall.allowedTCPPorts; + httpPort = matrixOldPath.services.hyperhive.swarm.matrix.httpPort; + in + builtins.elem httpPort ports + && matrixOldPath.containers.hive-matrix.bindMounts ? "/etc/matrix/register.token"; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA From 81b9ddd1891e6aeb2c155835674256474a402124 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 12:19:06 +0200 Subject: [PATCH 4/6] deploy: move the queue's callout identity out of swarm.nats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swarm.*` is what a hive needs to be a *client* of the swarm. For the queue that is the ports it answers on, the client id it is registered under, and the responder package. Whether this host mints its own auth-callout keypairs and where the seeds sit are decisions of the machine running it, so autoGenerateCallout, both seed files and both public keys move to `deploy.nats.*`. The two PUBLIC keys are the non-obvious half, so the reasoning is here rather than in a comment nobody re-reads. A public key looks swarm-wide, and docs/swarm/secrets.md says the user key "is published to every client that connects" — which is true and does not make it swarm config: peers RECEIVE it over the wire at connect time, they never configure it. What decides the placement is that its seed is unambiguously host-side, and splitting a keypair across two namespaces is worse than either placement. local-defaults.nix set `nats.autoGenerateCallout` from INSIDE `config.services.hyperhive.swarm = { ... }`, so the bare prefix meant `swarm.nats` and no full-path grep could see it. It moves out of that block into a `deploy` statement rather than being rewritten in place. swarm-controller.nix bound `natsCfg` and never used it — one mention, no bare pass, while its sibling bindings have 5 and 10. Pre-existing dead code, found by this slice's own alias sweep, removed with it. Four assertion messages and five doc lines named moved options by a short form (`nats.calloutUserSeedFile`) that is ambiguous now that both `swarm.nats` and `deploy.nats` exist; one opened with "services.hyperhive.swarm.nats has callout public keys", which the split makes false. Prose that names a VALUE rather than a path — the `nk` pipeline's `# -> calloutUserPublicKey` — is left bare on purpose. module-eval configures a hive through all five OLD paths and asserts the responder's delivery unit exists and interpolates the seed path. `autoGenerateCallout` is deliberately FALSE there: that is what makes the seed options decide `responderConfigured`, so the assertion is about the seeds rather than about the auto-mint branch. --- docs/swarm/secrets.md | 10 +-- nix/host-modules/deploy.nix | 31 +++++++++ nix/host-modules/local-defaults.nix | 14 ++-- nix/host-modules/swarm-controller.nix | 1 - nix/host-modules/swarm-nats.nix | 95 ++++++++++++++++----------- nix/module-eval.nix | 27 ++++++++ swarm-nats-auth/src/main.rs | 4 +- 7 files changed, 129 insertions(+), 53 deletions(-) diff --git a/docs/swarm/secrets.md b/docs/swarm/secrets.md index 4c1cdd4a..d02e51b8 100644 --- a/docs/swarm/secrets.md +++ b/docs/swarm/secrets.md @@ -14,7 +14,7 @@ than a convenience. The rule is about what must stay secret, not about credentials generally. **Public material is a value**: a certificate, or a public nkey like -`swarm.nats.calloutUserPublicKey`, is published to every client that connects, +`deploy.nats.calloutUserPublicKey`, is published to every client that connects, so the store is a perfectly good place for it. **The generator and the reader are usually in different containers.** They share @@ -54,7 +54,7 @@ neither is a renaming of the other. | the swarm collector's copy of its OIDC secret | `swarm-otel-oidc-secret.service` copies it from authelia's tree, **when authelia runs on this host** | `/var/lib/swarm-otel-oidc/.secret` inside the `swarm-otel` container | operator provides the file and names it in `swarm.otel.clientSecretFile` — the collector need not share a host with authelia | | authelia subject store | `swarmctl` and `swarm-authelia-bridge` | `users.yml` — one file, read and written by both | `swarmctl`, on the host that runs authelia | | wireguard private key | **the operator** — `wg genkey` | whatever `deploy.wireguard.privateKeyFile` names | always operator-provided; nothing generates this for you | -| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `nats.calloutUserSeedFile` / `nats.calloutIssuerSeedFile` | +| queue auth-callout nkeys (user seed + account seed) | `swarm-nats-callout-keys` first-boot unit, when `deploy.nats.autoGenerateCallout` is set | `/var/lib/swarm-nats-callout/{callout-user,issuer}.seed`, `0600` | operator mints both with `nk` and names them in `deploy.nats.calloutUserSeedFile` / `deploy.nats.calloutIssuerSeedFile` | | the secret store's own contents | openbao, on first `bao operator init` — **an operator action, not a unit** | inside the `swarm-bao` container, at its own `/var/lib/openbao`, kept across rebuilds by `ephemeral = false`. ⚠️ **Not a host path**: `nixos-container destroy swarm-bao` takes the raft data with it, so back up the container's tree, not `/var/lib/`. Only the store's TLS material (`/var/lib/swarm-bao-tls`) and its PKCS11 token (`/var/lib/swarm-bao-token`) are host-level | n/a — there is one store; a hive elsewhere is a _client_ of it and holds none of this | | the secret store's unseal material | the HSM/TPM under `deploy.bao.seal = "pkcs11"`; openbao itself under `"shamir"` | in the token; or held by whoever ran `bao operator init`, which is what `"shamir"` means and why it is stated rather than inferred | n/a — only the host running the store seals anything | @@ -90,7 +90,7 @@ not the definition of the case. ### Minting the queue's callout nkeys -`nats.autoGenerateCallout` mints both keypairs on the host before the queue +`deploy.nats.autoGenerateCallout` mints both keypairs on the host before the queue starts. It is on by default only under `singleHostSwarm` — the one topology where the queue, its responder and the operator are the same person. On every other topology, mint them yourself: @@ -102,8 +102,8 @@ nk -inkey callout-user.seed -pubout # → calloutUserPublicKey nk -inkey issuer.seed -pubout # → calloutIssuerPublicKey ``` -Keep both seeds at `0600` and name them in `calloutUserSeedFile` / -`calloutIssuerSeedFile`. Possession of the **issuer** seed is the authority to +Keep both seeds at `0600` and name them in `deploy.nats.calloutUserSeedFile` / +`deploy.nats.calloutIssuerSeedFile`. Possession of the **issuer** seed is the authority to admit anyone to the queue, so it belongs wherever the responder runs and nowhere else. diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 2ad3738f..757fd42c 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -98,6 +98,37 @@ in [ "services" "hyperhive" "deploy" "forgejo" "ci" "package" ] ) + # The queue's callout identity. What stays under `swarm.nats` is what the + # queue IS to every hive — the ports it answers on, the client id it is + # registered under, the responder package; these five are what the host + # running it decides. + # + # ⚠️ The two PUBLIC keys move with their seeds. A public key looks + # swarm-wide, and the user key genuinely is published to every client that + # connects — but peers RECEIVE it over the wire, they never configure it, + # and splitting a keypair across two namespaces is worse than either + # placement. `enable` moved earlier; its shim is the one above. + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "autoGenerateCallout" ] + [ "services" "hyperhive" "deploy" "nats" "autoGenerateCallout" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutUserPublicKey" ] + [ "services" "hyperhive" "deploy" "nats" "calloutUserPublicKey" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutIssuerPublicKey" ] + [ "services" "hyperhive" "deploy" "nats" "calloutIssuerPublicKey" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutUserSeedFile" ] + [ "services" "hyperhive" "deploy" "nats" "calloutUserSeedFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "nats" "calloutIssuerSeedFile" ] + [ "services" "hyperhive" "deploy" "nats" "calloutIssuerSeedFile" ] + ) + # The rest of the forge split. What stays under `swarm.forge` is what the # forge IS from any hive's point of view — its package, the names and ports # it answers on, the URLs it advertises, the client id it is registered diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 528f86de..8ef4b51d 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -78,14 +78,16 @@ in # the ⚠️ below gives about `swarm`. config.services.hyperhive.deploy.allSwarmServices = lib.mkDefault cfg.deploy.singleHostSwarm; + # The queue's auth-callout nkeys. Generating them is safe exactly + # when one operator owns both the queue and its responder, which is + # what this mode asserts. On any other topology the seeds have to + # reach whoever runs the responder, and minting them here would move + # that hand-off somewhere less visible rather than removing it. + config.services.hyperhive.deploy.nats.autoGenerateCallout = + lib.mkDefault cfg.deploy.singleHostSwarm; + config.services.hyperhive.swarm = { ca.autoConfigure = lib.mkDefault cfg.deploy.singleHostSwarm; - # The queue's auth-callout nkeys. Generating them is safe exactly - # when one operator owns both the queue and its responder, which is - # what this mode asserts. On any other topology the seeds have to - # reach whoever runs the responder, and minting them here would move - # that hand-off somewhere less visible rather than removing it. - nats.autoGenerateCallout = lib.mkDefault cfg.deploy.singleHostSwarm; # The controller's queue coordinates. Co-location is what makes these # derivable at all — loopback only reaches the queue when the queue is # here, and the minted client secret only exists on the host authelia diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index bb7d8ddd..b1f92ff4 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -32,7 +32,6 @@ let SWARMCTL_AUTHELIA_USERS_FILE = autheliaCfg.hostUsersFile; }; - natsCfg = config.services.hyperhive.swarm.nats; forgeCfg = config.services.hyperhive.swarm.forge; swarmDomain = config.services.hyperhive.swarm.domain; diff --git a/nix/host-modules/swarm-nats.nix b/nix/host-modules/swarm-nats.nix index 989522a6..605c5c4a 100644 --- a/nix/host-modules/swarm-nats.nix +++ b/nix/host-modules/swarm-nats.nix @@ -55,14 +55,17 @@ let # In auto mode the seeds are minted on this host before the container # starts, so they are configured by construction. responderConfigured = - cfg.autoGenerateCallout || (cfg.calloutUserSeedFile != "" && cfg.calloutIssuerSeedFile != ""); + deployCfg.nats.autoGenerateCallout + || (deployCfg.nats.calloutUserSeedFile != "" && deployCfg.nats.calloutIssuerSeedFile != ""); clientSecretSource = "${autheliaCfg.hostClientSecretDir}/${cfg.clientId}.secret"; introspectionUrl = "${toString autheliaUrl}/api/oidc/introspection"; # Where the responder's seeds actually come from. One name for two # origins, so everything downstream stops caring which mode it is in. - userSeedFile = if cfg.autoGenerateCallout then autoUserSeed else cfg.calloutUserSeedFile; - issuerSeedFile = if cfg.autoGenerateCallout then autoIssuerSeed else cfg.calloutIssuerSeedFile; + userSeedFile = + if deployCfg.nats.autoGenerateCallout then autoUserSeed else deployCfg.nats.calloutUserSeedFile; + issuerSeedFile = + if deployCfg.nats.autoGenerateCallout then autoIssuerSeed else deployCfg.nats.calloutIssuerSeedFile; # Seeds stay on the host at 0600 and never enter the container or the # store: only the responder needs them, and it reads them by @@ -281,6 +284,32 @@ in ''; }; + authPackage = lib.mkOption { + type = lib.types.package; + defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth"; + description = '' + The auth-callout responder package. + + ⚠️ Named `authPackage`, not `package`, on purpose: this module + deliberately has **no** `package` option for the server itself + (see the note above — upstream's `services.nats` resolves + `pkgs.nats-server` on its own), so a bare `package` here would + read as "the NATS package" and mean something else entirely. + ''; + }; + }; + + # What stays above is what the queue IS to every hive: the ports it answers + # on, the client id it is registered under, the responder package. The + # callout identity below is what the host running it decides — whether it + # mints its own keypairs, and where the seeds sit. `enable` already lives in + # ./deploy.nix, which also carries the renames. + # + # ⚠️ The two PUBLIC keys move with their seeds rather than staying: peers + # receive the user key over the wire when they connect (docs/swarm/secrets.md), + # they never configure it, and splitting a keypair across two namespaces is + # worse than either placement. + options.services.hyperhive.deploy.nats = { autoGenerateCallout = lib.mkOption { type = lib.types.bool; default = false; @@ -324,7 +353,7 @@ in — which is what makes a hive with no responder genuinely closed rather than merely gated. - Required when `enable` is set. + Required when `deploy.nats.enable` is set. ''; }; @@ -342,27 +371,13 @@ in and its whole job is to be widely known; the matching *seed* is the secret, is never named here, and reaches only the responder. - Required when `enable` is set. Without it the server has no - issuer to trust and no client can be authorized — which is the - fail-closed state described below, but arrived at by accident + Required when `deploy.nats.enable` is set. Without it the server + has no issuer to trust and no client can be authorized — which is + the fail-closed state described below, but arrived at by accident rather than on purpose, so it fails at eval instead. ''; }; - authPackage = lib.mkOption { - type = lib.types.package; - defaultText = lib.literalExpression "hyperhive.packages.\${system}.swarm-nats-auth"; - description = '' - The auth-callout responder package. - - ⚠️ Named `authPackage`, not `package`, on purpose: this module - deliberately has **no** `package` option for the server itself - (see the note above — upstream's `services.nats` resolves - `pkgs.nats-server` on its own), so a bare `package` here would - read as "the NATS package" and mean something else entirely. - ''; - }; - calloutUserSeedFile = lib.mkOption { type = lib.types.str; default = ""; @@ -412,11 +427,12 @@ in # Fail at EVAL, not at boot: a queue that comes up unable to # authenticate anyone presents as every client hanging, which is # several layers from "the operator never set the issuer". - assertion = cfg.autoGenerateCallout || cfg.calloutIssuerPublicKey != ""; + assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutIssuerPublicKey != ""; message = '' services.hyperhive.deploy.nats.enable requires - nats.calloutIssuerPublicKey — the public half of the account - nkey that signs user JWTs for this queue. + services.hyperhive.deploy.nats.calloutIssuerPublicKey — the + public half of the account nkey that signs user JWTs for this + queue. It is public and belongs in config; the matching seed is a secret and is delivered to the callout responder instead. See @@ -429,11 +445,12 @@ in # failure is the only place to catch that: the rendered config is # valid, the server starts, and the hole is invisible until # somebody connects. - assertion = cfg.autoGenerateCallout || cfg.calloutUserPublicKey != ""; + assertion = deployCfg.nats.autoGenerateCallout || deployCfg.nats.calloutUserPublicKey != ""; message = '' services.hyperhive.deploy.nats.enable requires - nats.calloutUserPublicKey — the public half of the user nkey - the auth-callout responder authenticates as. + services.hyperhive.deploy.nats.calloutUserPublicKey — the + public half of the user nkey the auth-callout responder + authenticates as. It is exempt from callout approval by design, which is exactly why it needs its own credential: a `users` entry with a name @@ -457,11 +474,11 @@ in # boolean is two places for a future edit to land in only one. assertion = responderConfigured; message = '' - services.hyperhive.swarm.nats has callout public keys but no + services.hyperhive.deploy.nats has callout public keys but no seed files: - nats.calloutUserSeedFile = "${cfg.calloutUserSeedFile}" - nats.calloutIssuerSeedFile = "${cfg.calloutIssuerSeedFile}" + deploy.nats.calloutUserSeedFile = "${deployCfg.nats.calloutUserSeedFile}" + deploy.nats.calloutIssuerSeedFile = "${deployCfg.nats.calloutIssuerSeedFile}" Each seed is the private half of the public key already set here — the server verifies with the public half, the responder @@ -469,8 +486,8 @@ in this queue with an auth-callout nobody answers, which refuses every client rather than degrading. - Set both seed paths, or set nats.autoGenerateCallout = true to - have this host mint all four. + Set both seed paths, or set deploy.nats.autoGenerateCallout = true + to have this host mint all four. ''; } { @@ -608,12 +625,12 @@ in # on fails the BUILD of every all-local hive. Upstream's own # description names the case: disable it when the config # includes other files. The check moves to server start. - validateConfig = !cfg.autoGenerateCallout; + validateConfig = !deployCfg.nats.autoGenerateCallout; settings = calloutBlocks { - userKey = cfg.calloutUserPublicKey; - issuerKey = cfg.calloutIssuerPublicKey; + userKey = deployCfg.nats.calloutUserPublicKey; + issuerKey = deployCfg.nats.calloutIssuerPublicKey; } // { # The monitoring endpoint, which is what the exporter below @@ -671,7 +688,7 @@ in # # `mkForce`: upstream defines ExecStart inside an `mkMerge`, so a # plain override conflicts rather than wins. - systemd.services.nats.serviceConfig.ExecStart = lib.mkIf cfg.autoGenerateCallout ( + systemd.services.nats.serviceConfig.ExecStart = lib.mkIf deployCfg.nats.autoGenerateCallout ( lib.mkForce "${pkgs.nats-server}/bin/nats-server -c ${runtimeWrapper}" ); @@ -757,7 +774,7 @@ in # nothing to deliver, and a copy that silently succeeds with a stale # or absent seed is worse than not running. after = - lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service" + lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service" # The third credential does not come from the generator above — it is # minted by authelia's FIRST BOOT, inside its own container. Ordering # after that container is necessary and NOT sufficient: the container @@ -765,7 +782,7 @@ in # has finished. The wait in the script is what actually closes it; # this only stops us spinning for the full timeout on every boot. ++ lib.optional deployCfg.authelia.enable "container@${autheliaCfg.machine}.service"; - requires = lib.optional cfg.autoGenerateCallout "swarm-nats-callout-keys.service"; + requires = lib.optional deployCfg.nats.autoGenerateCallout "swarm-nats-callout-keys.service"; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; @@ -820,7 +837,7 @@ in # a separate unit that needs the user seed: generating it inside would # trap it there and require a secret-export path back out — the exact # mechanism this is meant to avoid inventing. Only public halves cross. - systemd.services.swarm-nats-callout-keys = lib.mkIf cfg.autoGenerateCallout { + systemd.services.swarm-nats-callout-keys = lib.mkIf deployCfg.nats.autoGenerateCallout { description = "mint the swarm queue's auth-callout nkeys"; before = [ "container@swarm-nats.service" ]; wantedBy = [ "container@swarm-nats.service" ]; diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 301ce64c..82326028 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -104,6 +104,20 @@ let swarm.matrix.sso.clientSecretFile = "/etc/matrix/oidc.secret"; }; + # The queue's callout identity, fourth split slice. `autoGenerateCallout` is + # left FALSE on purpose: that is what makes the seed paths the thing deciding + # `responderConfigured`, so the assertion below is about the seeds rather + # than about the auto-mint branch. All five old paths are defined, so + # dropping any single shim entry fails the eval, not just the arms read. + natsOldPath = hive { + deploy.nats.enable = true; + swarm.nats.autoGenerateCallout = false; + swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -266,6 +280,19 @@ let builtins.elem httpPort ports && matrixOldPath.containers.hive-matrix.bindMounts ? "/etc/matrix/register.token"; } + { + # Reads the DELIVERY UNIT, not the options: `responderConfigured` gates + # whether it exists at all, and the seed path is interpolated into its + # script. A rename that resolved but stopped reaching the module would + # leave the responder with no credentials and this case would catch it. + name = "a config written against the pre-rename nats paths still delivers the responder's seeds"; + ok = + let + units = natsOldPath.systemd.services; + in + units ? swarm-nats-auth-secrets + && lib.hasInfix "/run/secrets/nats-user.seed" units.swarm-nats-auth-secrets.script; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA diff --git a/swarm-nats-auth/src/main.rs b/swarm-nats-auth/src/main.rs index 103ef8d0..a849419c 100644 --- a/swarm-nats-auth/src/main.rs +++ b/swarm-nats-auth/src/main.rs @@ -46,12 +46,12 @@ struct Args { nats_url: String, /// Path to the seed of the callout-exempt user this responder connects as. - /// Its public half is `services.hyperhive.swarm.nats.calloutUserPublicKey`. + /// Its public half is `services.hyperhive.deploy.nats.calloutUserPublicKey`. #[arg(long)] user_seed_file: PathBuf, /// Path to the account signing seed used to sign issued user JWTs. Its - /// public half is `services.hyperhive.swarm.nats.calloutIssuerPublicKey`. + /// public half is `services.hyperhive.deploy.nats.calloutIssuerPublicKey`. #[arg(long)] issuer_seed_file: PathBuf, From b6df0b4afcc2936908d53307462fcc459336c662 Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 13:09:55 +0200 Subject: [PATCH 5/6] deploy: move grafana's datasources, plugins and socket out of swarm.grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swarm.*` is what a hive needs to be a *client* of the swarm. For Grafana that is the package, the name it is served under, the port its `/metrics` is re-served on, and the OIDC client it is registered as. Where its datasources point, which plugins sit in its store path and the directory it shares a socket with nginx through are decisions of the machine running it, so datasourceUrl, logsDatasourceUrl, plugins and socketDir move to `deploy.grafana.*`. The two URLs are the interesting half. `swarm-grafana.nix`'s own summary sentence said what stays is "its package, domain, and wiring" — and both datasource URLs ARE wiring, so that sentence is rewritten with the move rather than left asserting the opposite of what the module does. They move because a URL's scope is the scope of what it ADDRESSES, not the fact that it is a URL: docs/swarm/services.md already said datasourceUrl "defaults to the store on this host, which is the only thing it can reach", because that store binds loopback. The doc argues the move. `socketDir` was already ruled host-side: the directory is shared between the host's nginx and the container, and it is the same shape as the options that moved in earlier slices. Declared in swarm-grafana.nix under the `deploy.*` path, following swarm-nats.nix; deploy.nix carries only the renames. One reader outside the options block is prose: `metricsPort` STAYS and cross-referenced `{option}services.hyperhive.swarm.grafana.socketDir`, which the split makes a pointer to another rendered page — nixosOptionsDoc emits swarm.md and deploy.md separately. Requalified, along with the one line in docs/swarm/services.md that named a mover. module-eval configures a hive through all four OLD paths and asserts a rendered effect: the host tmpfiles rule that creates the socket directory carries the fixture's custom path. The new paths evaluate fine without the shims, so dropping them has to read as a clean tree; all four are defined in the fixture, so removing any single shim fails the eval rather than only the one the assertion reads. --- docs/swarm/services.md | 2 +- nix/host-modules/deploy.nix | 16 ++++ nix/host-modules/swarm-grafana.nix | 130 ++++++++++++++++------------- nix/module-eval.nix | 20 +++++ 4 files changed, 110 insertions(+), 58 deletions(-) diff --git a/docs/swarm/services.md b/docs/swarm/services.md index 0208b596..d1ff3fb2 100644 --- a/docs/swarm/services.md +++ b/docs/swarm/services.md @@ -116,7 +116,7 @@ See `retentionPeriod` below before leaving it at its default. | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `deploy.victoriametrics.retentionPeriod` | Default `5y`. Lower it once you have measured how fast this swarm actually fills a disk — the default is deliberately generous because too-short silently discards history you cannot get back. | | `swarm.grafana.oidc.role` | Default `Admin` for everyone who logs in. Lower to `Viewer`/`Editor` if the swarm grows operators who should not be able to reconfigure Grafana. | -| `swarm.grafana.datasourceUrl` | Only if you front VictoriaMetrics with something else. It defaults to the store on this host, which is the only thing it can reach. | +| `deploy.grafana.datasourceUrl` | Only if you front VictoriaMetrics with something else. It defaults to the store on this host, which is the only thing it can reach. | **Logging in.** Grafana is behind swarm SSO, so the accounts are the authelia ones — there is no separate Grafana password, and the local diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 757fd42c..92ae1978 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -41,6 +41,22 @@ in [ "services" "hyperhive" "swarm" "grafana" "enable" ] [ "services" "hyperhive" "deploy" "grafana" "enable" ] ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "grafana" "socketDir" ] + [ "services" "hyperhive" "deploy" "grafana" "socketDir" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "grafana" "datasourceUrl" ] + [ "services" "hyperhive" "deploy" "grafana" "datasourceUrl" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "grafana" "logsDatasourceUrl" ] + [ "services" "hyperhive" "deploy" "grafana" "logsDatasourceUrl" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "grafana" "plugins" ] + [ "services" "hyperhive" "deploy" "grafana" "plugins" ] + ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "victoriametrics" "enable" ] [ "services" "hyperhive" "deploy" "victoriametrics" "enable" ] diff --git a/nix/host-modules/swarm-grafana.nix b/nix/host-modules/swarm-grafana.nix index 5c083c86..3ec186e0 100644 --- a/nix/host-modules/swarm-grafana.nix +++ b/nix/host-modules/swarm-grafana.nix @@ -92,7 +92,7 @@ let # the OIDC secret above, whose other reader is authelia's container. secretKeyPath = "/var/lib/grafana-secret/secret_key"; - socketPath = "${cfg.socketDir}/grafana.sock"; + socketPath = "${deployCfg.grafana.socketDir}/grafana.sock"; # Both static NixOS ids, and both checked rather than assumed: nginx has # `ids.gids.nginx = 60`, Grafana has `ids.uids.grafana = 196` — but there is @@ -111,8 +111,12 @@ in # `enable` moved to `services.hyperhive.deploy.grafana.enable` — see # ./deploy.nix. Whether this host runs the swarm's Grafana is a # deployment decision, and `swarm.*` has to be identical on every host. - # What stays here is what the service IS: its package, domain, and - # wiring. + # What stays here is what the service IS to every hive: its package, the + # name it is served under, the port its `/metrics` is re-served on, and + # the OIDC client it is registered as. What the host running it decides — + # where its datasources point, which plugins are in its store path, and + # the socket directory it shares with nginx — is below, under + # `deploy.grafana`. options.services.hyperhive.swarm.grafana = { package = lib.mkOption { type = lib.types.package; @@ -148,25 +152,6 @@ in ''; }; - socketDir = lib.mkOption { - type = lib.types.str; - default = "/run/swarm-grafana"; - description = '' - Directory holding the unix socket Grafana serves on, shared between - the host (where nginx runs) and the container (where Grafana runs). - - ⚠️ **Grafana takes no TCP port at all, and that is the point.** Every - swarm service container shares the host's network namespace, so a - port is a swarm-wide resource that two modules can silently both - claim — which is exactly what happened: Grafana defaulted to - upstream's 3000, so does the forge, and `grafana.` - served the forge with no bind error and nothing in any log. - - A socket has a path, and a path collision is a build-time conflict - rather than a runtime coin toss. - ''; - }; - metricsPort = lib.mkOption { type = lib.types.port; default = 9095; @@ -175,7 +160,7 @@ in `/metrics`, and nothing else, so the swarm's collector can scrape it. ⚠️ **This is nginx's port, not Grafana's.** Grafana still claims none — - see {option}`services.hyperhive.swarm.grafana.socketDir` for why that + see {option}`services.hyperhive.deploy.grafana.socketDir` for why that matters. A prometheus scrape target is a `host:port`, and it cannot address a unix socket; rather than undo the socket decision, the one endpoint a scraper needs gets a listener of its own. @@ -194,34 +179,6 @@ in ''; }; - datasourceUrl = lib.mkOption { - type = lib.types.str; - default = "http://127.0.0.1:${toString vmCfg.port}"; - defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"''; - description = '' - Where the provisioned datasource points. Defaults to the metrics - store on this host, which is the only place it can be: that store - binds loopback, so a Grafana somewhere else could not reach it - anyway. Set explicitly if a deployment fronts VictoriaMetrics with - something that does listen wider. - ''; - }; - - logsDatasourceUrl = lib.mkOption { - type = lib.types.str; - default = "http://127.0.0.1:${toString vlCfg.port}"; - defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victorialogs.port}"''; - description = '' - Where the provisioned logs datasource points. Same reasoning as - {option}`services.hyperhive.swarm.grafana.datasourceUrl`: the store - binds loopback, so a Grafana elsewhere could not reach it anyway. - - Provisioned unconditionally, like the metrics datasource — the store - being off is a deployment choice rather than a reason to withhold the - connection, and an operator whose logs live elsewhere sets this. - ''; - }; - oidc = { clientId = lib.mkOption { type = lib.types.str; @@ -257,6 +214,65 @@ in }; }; + }; + + # What stays above is what the service IS to every hive. What the host + # running it decides is here: where its datasources point, which plugins sit + # in its store path, and the directory it shares a socket with nginx + # through. `enable` already lives in ./deploy.nix, which also carries the + # renames. + # + # ⚠️ Both datasource URLs are wiring and still move. A URL's scope is the + # scope of what it ADDRESSES, not the fact that it is a URL: both stores + # bind loopback, so these can only ever mean "on this host". + options.services.hyperhive.deploy.grafana = { + socketDir = lib.mkOption { + type = lib.types.str; + default = "/run/swarm-grafana"; + description = '' + Directory holding the unix socket Grafana serves on, shared between + the host (where nginx runs) and the container (where Grafana runs). + + ⚠️ **Grafana takes no TCP port at all, and that is the point.** Every + swarm service container shares the host's network namespace, so a + port is a swarm-wide resource that two modules can silently both + claim — which is exactly what happened: Grafana defaulted to + upstream's 3000, so does the forge, and `grafana.` + served the forge with no bind error and nothing in any log. + + A socket has a path, and a path collision is a build-time conflict + rather than a runtime coin toss. + ''; + }; + + datasourceUrl = lib.mkOption { + type = lib.types.str; + default = "http://127.0.0.1:${toString vmCfg.port}"; + defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victoriametrics.port}"''; + description = '' + Where the provisioned datasource points. Defaults to the metrics + store on this host, which is the only place it can be: that store + binds loopback, so a Grafana somewhere else could not reach it + anyway. Set explicitly if a deployment fronts VictoriaMetrics with + something that does listen wider. + ''; + }; + + logsDatasourceUrl = lib.mkOption { + type = lib.types.str; + default = "http://127.0.0.1:${toString vlCfg.port}"; + defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString services.hyperhive.swarm.victorialogs.port}"''; + description = '' + Where the provisioned logs datasource points. Same reasoning as + {option}`services.hyperhive.deploy.grafana.datasourceUrl`: the store + binds loopback, so a Grafana elsewhere could not reach it anyway. + + Provisioned unconditionally, like the metrics datasource — the store + being off is a deployment choice rather than a reason to withhold the + connection, and an operator whose logs live elsewhere sets this. + ''; + }; + plugins = lib.mkOption { type = lib.types.listOf lib.types.package; default = [ @@ -428,7 +444,7 @@ in # because the host has no `grafana` account — Grafana lives in the # container, and only the number crosses that boundary. systemd.tmpfiles.rules = [ - "d ${cfg.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" + "d ${deployCfg.grafana.socketDir} 0750 ${toString grafanaUid} ${toString nginxGid} - -" ]; # The secret delivery. It runs on the HOST because that is the only place @@ -513,8 +529,8 @@ in # ownership) and nspawn mounts it in — the same shape the per-agent # `/run/hive-agent//web.sock` already uses. bindMounts = { - ${cfg.socketDir} = { - hostPath = cfg.socketDir; + ${deployCfg.grafana.socketDir} = { + hostPath = deployCfg.grafana.socketDir; isReadOnly = false; }; } @@ -636,7 +652,7 @@ in # hive with no plugins, manual installation is impossible anyway" # — reasoned only about plugins a PERSON installs. Upstream # installs some itself, and those are the ones that went missing. - declarativePlugins = cfg.plugins; + declarativePlugins = deployCfg.grafana.plugins; settings = { server = { @@ -716,7 +732,7 @@ in name = "VictoriaMetrics"; type = "prometheus"; uid = datasourceUid; - url = cfg.datasourceUrl; + url = deployCfg.grafana.datasourceUrl; access = "proxy"; isDefault = true; } @@ -728,7 +744,7 @@ in # fails at use, far from anything naming this file. type = "victoriametrics-logs-datasource"; uid = logsDatasourceUid; - url = cfg.logsDatasourceUrl; + url = deployCfg.grafana.logsDatasourceUrl; access = "proxy"; # Exactly one datasource may claim this, and metrics has it: # two defaults is a coin toss over which one an untyped panel diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 82326028..3c5cde46 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -118,6 +118,14 @@ let swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; }; + grafanaOldPath = hive { + deploy.grafana.enable = true; + swarm.grafana.socketDir = "/run/test-grafana-sock"; + swarm.grafana.datasourceUrl = "http://127.0.0.1:19999"; + swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998"; + swarm.grafana.plugins = [ ]; + }; + baoPkcs11 = hive { deploy.bao.enable = true; deploy.bao.seal = "pkcs11"; @@ -293,6 +301,18 @@ let units ? swarm-nats-auth-secrets && lib.hasInfix "/run/secrets/nats-user.seed" units.swarm-nats-auth-secrets.script; } + { + # Reads the host's tmpfiles rules, not the options: the socket directory + # nginx and the container share is created there, so a rename that + # resolved but stopped reaching the module would leave the gateway + # proxying to a path nothing creates. All four old paths are defined in + # the fixture, so removing any single shim fails the eval rather than + # only the one this assertion reads. + name = "a config written against the pre-rename grafana paths still creates the socket directory"; + ok = lib.any ( + rule: lib.hasInfix "/run/test-grafana-sock" rule + ) grafanaOldPath.systemd.tmpfiles.rules; + } { # The gateway's per-name issuer choice. If this ever collapses to a # constant, every swarm-service vhost serves a certificate its CA From b7c5067f7986a76090272832fb1e1fa70ddab73f Mon Sep 17 00:00:00 2001 From: atlas Date: Mon, 7 Sep 2026 13:31:34 +0200 Subject: [PATCH 6/6] deploy: move the controller's socket and credentials out of swarm.controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `swarm.*` is what a hive needs to be a *client* of the swarm. For the controller that is the quick-links it publishes, the client id it is registered under, how stale a snapshot may read, and its packages. Where its socket lives and where the three credentials it reads sit are decisions of the machine running it, so socketPath, forgeTokenFile, authBridgeUrl and queue.clientSecretFile move to `deploy.swarm-controller.*`. ⚠️ `deploy.swarm-controller`, not `deploy.hive-controller`. Both exist on main — the latter is hive-c0re's namespace, where `tls.*` lives — and a definition on the wrong one lands on a live unrelated option instead of failing. The shim table is anchored on both sides for that reason. `queue.clientSecretFile` is a nested field, not a top-level option: it moves out of `queue` and leaves natsUrl and tokenEndpoint behind, so the far side needs `deploy.swarm-controller.queue.clientSecretFile`. That is the `forge.sso` split shape, applied again rather than reinvented — the endpoints a client dials are swarm-wide, the secret beside them is a path on one host. local-defaults.nix set that secret from INSIDE `config.services.hyperhive.swarm = { ... }`, where a bare `controller.` prefix means `swarm.controller`. Left there it would still resolve — through the rename — and warn on every evaluation of a single-host swarm, which is the same defect an earlier slice fixed for the matrix module. It moves out into its own `deploy` statement beside the controller's `enable`. `queue.natsUrl` stays bare: it is a stayer. swarm-ui.nix read `socketPath` through its own `controllerCfg` alias. With that repointed the binding had no reader left, so it goes. Unlike the dead `natsCfg` an earlier slice removed, this one was live until this commit — the move orphaned it. `deployCfg` was already bound there. Prose that named a moved option by its full path is requalified: the assertion message for the queue secret (operator-facing, and inside `config` where an options-block sweep would miss it) and `swarm-controller/README.md`'s socket-path line. Left bare on purpose: `docs/swarm/README.md` and the README's own "never point `socketPath` at a directory that carries anything else" both name the FEATURE, not a path. module-eval configures a hive through all four OLD paths and asserts a rendered effect for EACH of them — the unit's socket env, its auth-bridge env, and both credential paths in `LoadCredential`. Asserting all four rather than one means a rename that resolves but stops reaching the module is caught per-option, not only where a single assertion happens to look. --- nix/host-modules/deploy.nix | 16 ++++ nix/host-modules/local-defaults.nix | 14 ++- nix/host-modules/swarm-controller.nix | 128 +++++++++++++++----------- nix/host-modules/swarm-ui.nix | 5 +- nix/module-eval.nix | 23 +++++ swarm-controller/README.md | 2 +- 6 files changed, 126 insertions(+), 62 deletions(-) diff --git a/nix/host-modules/deploy.nix b/nix/host-modules/deploy.nix index 92ae1978..f1964fc9 100644 --- a/nix/host-modules/deploy.nix +++ b/nix/host-modules/deploy.nix @@ -69,6 +69,22 @@ in [ "services" "hyperhive" "swarm" "controller" "enable" ] [ "services" "hyperhive" "deploy" "swarm-controller" "enable" ] ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "controller" "socketPath" ] + [ "services" "hyperhive" "deploy" "swarm-controller" "socketPath" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "controller" "forgeTokenFile" ] + [ "services" "hyperhive" "deploy" "swarm-controller" "forgeTokenFile" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "controller" "authBridgeUrl" ] + [ "services" "hyperhive" "deploy" "swarm-controller" "authBridgeUrl" ] + ) + (lib.mkRenamedOptionModule + [ "services" "hyperhive" "swarm" "controller" "queue" "clientSecretFile" ] + [ "services" "hyperhive" "deploy" "swarm-controller" "queue" "clientSecretFile" ] + ) (lib.mkRenamedOptionModule [ "services" "hyperhive" "swarm" "ui" "enable" ] [ "services" "hyperhive" "deploy" "swarm-ui" "enable" ] diff --git a/nix/host-modules/local-defaults.nix b/nix/host-modules/local-defaults.nix index 8ef4b51d..afa94b7b 100644 --- a/nix/host-modules/local-defaults.nix +++ b/nix/host-modules/local-defaults.nix @@ -112,9 +112,6 @@ in controller.queue.natsUrl = lib.mkIf cfg.deploy.singleHostSwarm ( lib.mkDefault "nats://127.0.0.1:${toString config.services.hyperhive.swarm.nats.port}" ); - controller.queue.clientSecretFile = lib.mkIf cfg.deploy.singleHostSwarm ( - lib.mkDefault "${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/swarm-controller.secret" - ); }; # The controller is asserted by the MODE and by nothing else. Its own @@ -131,4 +128,15 @@ in # does not apply here — that one is about two definitions of `swarm` # itself, and this is a different top-level path. config.services.hyperhive.deploy.swarm-controller.enable = lib.mkDefault cfg.deploy.singleHostSwarm; + + # Same reason, one option later: the queue secret is a path on THIS host, + # so it moved to `deploy.*` with the rest of the controller's credentials. + # It has to sit out here rather than in the `swarm` attrset above — a bare + # `controller.` prefix in there means `swarm.controller`, which is now only + # a rename shim, so the definition would still resolve and warn on every + # evaluation of a single-host swarm. + config.services.hyperhive.deploy.swarm-controller.queue.clientSecretFile = + lib.mkIf cfg.deploy.singleHostSwarm ( + lib.mkDefault "${config.services.hyperhive.swarm.authelia.hostClientSecretDir}/swarm-controller.secret" + ); } diff --git a/nix/host-modules/swarm-controller.nix b/nix/host-modules/swarm-controller.nix index b1f92ff4..fe1c505e 100644 --- a/nix/host-modules/swarm-controller.nix +++ b/nix/host-modules/swarm-controller.nix @@ -91,15 +91,15 @@ let # (`queueLocal`) is gone — its coordinates are options with assertions, # so a split-host controller fails at eval instead of silently losing # its identity. Forge access still has nothing to do with where authelia - # and NATS run, only with whether `cfg.forgeTokenFile` resolves to a real - # file — which `forgeTokenFile`'s own default already handles + # and NATS run, only with whether `forgeTokenFile` resolves to a real + # file — which that option's own default already handles # (forge-local vs. operator-copied). Gating a second time here would # just repeat that option's own logic under a different name. # # ⚠️ The asymmetry is deliberate: the queue is REQUIRED for a controller # and the forge is not, so absence is an error for one and a supported # shape for the other. - forgeEnv = lib.optionalAttrs (cfg.forgeTokenFile != null) { + forgeEnv = lib.optionalAttrs (deployCfg.swarm-controller.forgeTokenFile != null) { SWARM_CONTROLLER_FORGE_URL = "https://${forgeCfg.domain}"; # Same `%d` shape as the queue secret above — root reads the plaintext # at unit start, the daemon's own user sees a 0400 copy. @@ -137,8 +137,8 @@ let # is satisfied by THIS daemon's existing queue OIDC identity # (`queueEnv` above): "one identity per principal" already covers this, # so there is nothing new to mint or copy, just the bridge's address. - authBridgeEnv = lib.optionalAttrs (cfg.authBridgeUrl != null) { - SWARM_CONTROLLER_AUTH_BRIDGE_URL = cfg.authBridgeUrl; + authBridgeEnv = lib.optionalAttrs (deployCfg.swarm-controller.authBridgeUrl != null) { + SWARM_CONTROLLER_AUTH_BRIDGE_URL = deployCfg.swarm-controller.authBridgeUrl; }; # The swarm's own display name, for `GET /api/swarm` (swarm-ui's chrome). @@ -268,27 +268,6 @@ in ''; }; - socketPath = lib.mkOption { - type = lib.types.str; - default = "/run/swarm-controller/controller.sock"; - description = '' - Unix socket the daemon serves on, and the path the gateway's nginx - proxies to. - - The **directory** is the access control here, not the socket mode: - the socket itself is `0666` (nginx runs as another user, and - `connect(2)` needs write), exactly as hive-c0re publishes the - per-agent sockets. What keeps that safe is that the directory holds - exactly one socket and is traverse-only (`0751`) for everyone else. - Moving this path under a directory that carries anything else — - `/run/hyperhive`, which holds the host admin socket, above all — - exposes whatever else lives there to the same reachability. - - Changing this therefore means re-checking what else lives in the - new directory, not just the daemon. - ''; - }; - staleAfterSeconds = lib.mkOption { type = lib.types.ints.positive; default = 120; @@ -340,27 +319,6 @@ in correct for a remote provider as long as that URL is set. ''; }; - - clientSecretFile = lib.mkOption { - type = lib.types.str; - default = ""; - example = "/var/lib/secrets/swarm-controller-queue.secret"; - description = '' - Path on **this** host holding the plaintext of the controller's - OAuth2 client secret. A path, never a value: the secret would - otherwise land in the world-readable nix store. - - The controller cannot mint its own — minting happens inside - authelia's state directory during its first boot — so away from - that host the operator places the secret and names it here. - `singleHostSwarm` points this at the minted file, which - is exactly the case where one exists locally. - - Read by `LoadCredential`, so it needs to be readable by root at - unit start and nothing more; the daemon's own user never sees - the original path. - ''; - }; }; links = lib.mkOption { @@ -410,6 +368,40 @@ in ''; }; + }; + + # What stays above is what the controller IS to the swarm: the quick-links + # it publishes, the client id it is registered under, how stale a snapshot + # may read, and its packages. What the host running it decides is here — + # where its socket lives, and the three credentials it reads off local + # disk. `enable` already lives in ./deploy.nix, which also carries the + # renames. + # + # ⚠️ `deploy.swarm-controller`, NOT `deploy.hive-controller`. Both exist: + # the latter is hive-c0re's namespace, where `tls.*` lives. A definition on + # the wrong one lands on a live, unrelated option rather than failing. + options.services.hyperhive.deploy.swarm-controller = { + socketPath = lib.mkOption { + type = lib.types.str; + default = "/run/swarm-controller/controller.sock"; + description = '' + Unix socket the daemon serves on, and the path the gateway's nginx + proxies to. + + The **directory** is the access control here, not the socket mode: + the socket itself is `0666` (nginx runs as another user, and + `connect(2)` needs write), exactly as hive-c0re publishes the + per-agent sockets. What keeps that safe is that the directory holds + exactly one socket and is traverse-only (`0751`) for everyone else. + Moving this path under a directory that carries anything else — + `/run/hyperhive`, which holds the host admin socket, above all — + exposes whatever else lives there to the same reachability. + + Changing this therefore means re-checking what else lives in the + new directory, not just the daemon. + ''; + }; + forgeTokenFile = lib.mkOption { type = lib.types.nullOr lib.types.str; # The forge has no `enable` of its own to check — neither half of @@ -465,13 +457,37 @@ in No new credential to configure: the bearer token presented to the bridge is minted from THIS daemon's own existing queue OIDC - identity (`queue.*` above) — "one identity per principal" + identity — its endpoints stay in `swarm.controller.queue`, its secret is + `queue.clientSecretFile` below — "one identity per principal" already covers it. `null` means no agent-identity support: `CreateIdentity` jobs fail with a clear "no auth bridge configured here" error rather than the daemon refusing to start, the same graceful-absence shape `forgeTokenFile` uses. ''; }; + + queue = { + clientSecretFile = lib.mkOption { + type = lib.types.str; + default = ""; + example = "/var/lib/secrets/swarm-controller-queue.secret"; + description = '' + Path on **this** host holding the plaintext of the controller's + OAuth2 client secret. A path, never a value: the secret would + otherwise land in the world-readable nix store. + + The controller cannot mint its own — minting happens inside + authelia's state directory during its first boot — so away from + that host the operator places the secret and names it here. + `singleHostSwarm` points this at the minted file, which + is exactly the case where one exists locally. + + Read by `LoadCredential`, so it needs to be readable by root at + unit start and nothing more; the daemon's own user never sees + the original path. + ''; + }; + }; }; config = lib.mkIf (config.services.hyperhive.enable && deployCfg.swarm-controller.enable) { @@ -555,10 +571,10 @@ in ''; } { - assertion = cfg.queue.clientSecretFile != ""; + assertion = deployCfg.swarm-controller.queue.clientSecretFile != ""; message = '' - services.hyperhive.swarm.controller.queue.clientSecretFile is - unset. + services.hyperhive.deploy.swarm-controller.queue.clientSecretFile + is unset. It defaults to the file authelia's first-boot generator mints, which only exists when authelia runs on this host. Elsewhere the @@ -593,9 +609,11 @@ in # that race on a real boot — two seconds more and it would have hit # `start-limit-hit`, which does not self-heal. LoadCredential = [ - "queue-client.secret:${cfg.queue.clientSecretFile}" + "queue-client.secret:${deployCfg.swarm-controller.queue.clientSecretFile}" ] - ++ lib.optional (cfg.forgeTokenFile != null) "forge-token:${cfg.forgeTokenFile}"; + ++ lib.optional ( + deployCfg.swarm-controller.forgeTokenFile != null + ) "forge-token:${deployCfg.swarm-controller.forgeTokenFile}"; # The placeholder default that makes the above non-fatal. # `LoadCredential=` takes priority over `SetCredential=`, so this is @@ -685,7 +703,7 @@ in # broken rather than lighter; forge's and the auth bridge's are # genuinely optional and stay gated on their own option resolving. environment = { - SWARM_CONTROLLER_SOCKET = cfg.socketPath; + SWARM_CONTROLLER_SOCKET = deployCfg.swarm-controller.socketPath; # The swarm's hive directory, JSON-encoded — the full directory # (this daemon has no "self" hive to exclude, unlike # `swarm.peerHives`, `swarm.hives` minus this hive) rather than @@ -750,7 +768,7 @@ in description = "watch the swarm controller's queue credential"; wantedBy = [ "multi-user.target" ]; pathConfig = { - PathChanged = cfg.queue.clientSecretFile; + PathChanged = deployCfg.swarm-controller.queue.clientSecretFile; Unit = "swarm-controller-credential.service"; }; }; diff --git a/nix/host-modules/swarm-ui.nix b/nix/host-modules/swarm-ui.nix index 08068f55..dd389558 100644 --- a/nix/host-modules/swarm-ui.nix +++ b/nix/host-modules/swarm-ui.nix @@ -17,7 +17,6 @@ let deployCfg = config.services.hyperhive.deploy; gatewayCfg = config.services.hyperhive.gateway; autheliaCfg = config.services.hyperhive.swarm.authelia; - controllerCfg = config.services.hyperhive.swarm.controller; # Same stylix auto-theming `hive-c0re/theme.nix` gives the dashboard — # this UI was left out of that overlay entirely (an operator with a @@ -211,7 +210,7 @@ in # path swarm-controller registered a route at is the path # nginx forwards, no prefix-stripping to keep in sync by hand. "/api/" = { - proxyPass = "http://unix:${controllerCfg.socketPath}:"; + proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; extraConfig = swarmAuthRequest; }; # ⚠️ THE ONE LOCATION ON THIS VHOST WITH NO `swarmAuthRequest`, @@ -233,7 +232,7 @@ in # a future `/webhook/` does not inherit the # bypass by living under a shared prefix. "/webhook/forge/" = { - proxyPass = "http://unix:${controllerCfg.socketPath}:"; + proxyPass = "http://unix:${deployCfg.swarm-controller.socketPath}:"; }; # Swagger UI: same "nginx hosts the themed dist straight from # the store, only /api/openapi.json is dynamic" shape as the diff --git a/nix/module-eval.nix b/nix/module-eval.nix index 3c5cde46..29e23a1b 100644 --- a/nix/module-eval.nix +++ b/nix/module-eval.nix @@ -118,6 +118,14 @@ let swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; }; + controllerOldPath = hive { + deploy.swarm-controller.enable = true; + swarm.controller.socketPath = "/run/test-ctrl/ctrl.sock"; + swarm.controller.forgeTokenFile = "/run/secrets/ctrl-forge.token"; + swarm.controller.authBridgeUrl = "http://127.0.0.1:19097"; + swarm.controller.queue.clientSecretFile = "/run/secrets/ctrl-queue.secret"; + }; + grafanaOldPath = hive { deploy.grafana.enable = true; swarm.grafana.socketDir = "/run/test-grafana-sock"; @@ -308,6 +316,21 @@ let # proxying to a path nothing creates. All four old paths are defined in # the fixture, so removing any single shim fails the eval rather than # only the one this assertion reads. + # All FOUR movers are asserted as rendered effects rather than as option + # values, so a rename that resolved but stopped reaching the module is + # caught per-option instead of only where one assertion happens to look. + name = "a config written against the pre-rename swarm-controller paths still reaches the unit"; + ok = + let + u = controllerOldPath.systemd.services.swarm-controller; + creds = u.serviceConfig.LoadCredential; + in + u.environment.SWARM_CONTROLLER_SOCKET == "/run/test-ctrl/ctrl.sock" + && u.environment.SWARM_CONTROLLER_AUTH_BRIDGE_URL == "http://127.0.0.1:19097" + && lib.any (c: lib.hasInfix "/run/secrets/ctrl-queue.secret" c) creds + && lib.any (c: lib.hasInfix "/run/secrets/ctrl-forge.token" c) creds; + } + { name = "a config written against the pre-rename grafana paths still creates the socket directory"; ok = lib.any ( rule: lib.hasInfix "/run/test-grafana-sock" rule diff --git a/swarm-controller/README.md b/swarm-controller/README.md index b9900d14..745ee52b 100644 --- a/swarm-controller/README.md +++ b/swarm-controller/README.md @@ -24,7 +24,7 @@ The hive-gateway's nginx is the only intended client and reaches the socket through a bind-mount. A listener that is never bound to an address cannot be reached from off-host by mistake. -The socket path is `services.hyperhive.swarm.controller.socketPath`, default +The socket path is `services.hyperhive.deploy.swarm-controller.socketPath`, default `/run/swarm-controller/controller.sock`, exported to the process as `SWARM_CONTROLLER_SOCKET`.