# The swarm's directory: one entry per hive, **including this one**, # identical on every host in the swarm. `services.hyperhive.hiveName` # says which entry is us. # # Why a directory rather than a per-host peer list: every field here is # intrinsic to the hive it describes — none of them says anything about # the *pair*. A list where every field is intrinsic is a directory each # host was keeping its own copy of, which is O(n²) duplication that # deduplicates without loss. It is also a correctness gain: two hosts # could hold different endpoints for the same third hive and nothing # detected it. One entry per hive makes that unrepresentable. # # Consumed by swarm-controller's own hive directory (its `/api/hives`, # swarm-controller.nix) and the mesh in ./swarm-wireguard.nix. The mesh # lives there rather than here because bringing up an interface is # host networking rather than swarm bookkeeping, and a host that runs # no hive still needs it. { lib, config, ... }: let cfg = config.services.hyperhive; swarmCfg = cfg.swarm; deployCfg = cfg.deploy; hiveNames = lib.attrNames swarmCfg.hives; nameGuards = import ./lib/name-guards.nix { inherit lib; }; # Names no hive may BE, and words no hive name may CONTAIN. Both files # explain their own admission rules; the second says why they are not one # list. They are applied HERE rather than in the module that happens to # consume them, because a hive name composes identifiers whatever else is # enabled — gating them on another module's `enable` would leave a swarm # without that module with no check at all. reservedNames = import ../reserved-names.nix; reservedFragments = import ../reserved-hive-fragments.nix; # Subjects the store's cert-auth roles accept. A LIST, not one string: cert # auth trusts the CA and `allowed_common_names` is the whole narrowing, so # every role added beside these widens what a hive name must not collide # with. A hive's own leaf carries its name as the CN, so a hive named after # one of these presents a certificate that role accepts. certAuthCns = [ deployCfg.bao.controllerCommonName deployCfg.bao.secretPublisherCommonName deployCfg.bao.matrixCtlCommonName deployCfg.bao.grafanaOidcCommonName deployCfg.bao.otelOidcCommonName deployCfg.bao.forwarderOidcCommonName deployCfg.bao.servicesIssuerCommonName deployCfg.bao.natsCommonName deployCfg.bao.granterCommonName deployCfg.bao.natsAuthCommonName deployCfg.bao.ciRunnerCommonName deployCfg.bao.logsAuthCommonName deployCfg.bao.metricsAuthCommonName deployCfg.bao.reviewClassifierCommonName deployCfg.bao.builderCommonName deployCfg.bao.claudeLoginCommonName deployCfg.bao.forgePublisherCommonName ] # The per-hive subjects, spelled out per hive rather than as the prefix. # The prefix alone would reserve the wrong string: the role for hive # `h` accepts `-h`, so the name a hive must not BE is that composed # spelling, and a hive named it would present a leaf that role accepts. # # Every spelling is reserved for every declared hive, including the one the # CN belongs to — a hive cannot be named after its own reader either. ++ lib.concatMap (hive: [ "${deployCfg.bao.queueAgentCommonNamePrefix}-${hive}" "${deployCfg.bao.hiveLeafCommonNamePrefix}-${hive}" ]) hiveNames; # Public hostnames of the swarm's own services, in declaration order. # `serviceDomains` below is this set sorted + deduplicated. # # ⚠️ These are NOT required to be under `swarm.domain`. An earlier # revision asserted that, reasoning that the services issuer is # constrained to the swarm's tree — but the store's `swarm-services` # role admits exactly the **configured names** (./swarm-bao.nix), so any # configured name is issuable. The assertion encoded an intended shape, # not a property of the code, and it rejected the supported migration # path: a hive pinning its old `forge.` while joining a swarm # at a different apex. # # Every name is listed whether or not this host runs the service: the # store's issuing role is built from this list on the store's host and # has to admit a leaf for any gateway in the swarm. What one gateway's # leaf carries is `localServiceDomains`. # # Membership is also what `gateway.lib.tlsFor` consults to pick the # services leaf over the hive one, so a gateway-published swarm name # missing here is served the hive certificate, which cannot cover a name # under a different apex. # # `swarm.nats.domain` is deliberately absent: its `/metrics` vhost names # the queue's own leaf instead (./swarm-nats.nix), so a host running only # the queue fronts no services name and never waits for that leaf. # # So are the hives' `dashboardDomains`: each is a SAN on its own hive's # leaf, which that hive issues with the identity it already holds. serviceDomains' = [ swarmCfg.forge.domain swarmCfg.matrix.gatewayHost swarmCfg.authelia.domain # The swarm UI's apex is a sibling of the other names, not a parent: # no CA in the hierarchy issues for it implicitly. swarmCfg.domain swarmCfg.grafana.domain swarmCfg.victoriametrics.domain swarmCfg.otel.domain swarmCfg.victorialogs.domain # The store's browser UI, not the store: `swarm.bao.domain` is mTLS and # never gets a gateway certificate. swarmCfg.bao.ui.domain ]; # Hives whose entry still carries the removed `certFingerprint`. Scanned # here, at top level, because that is the only place an assertion about a # submodule field can actually live — see the option's own comment. pinnedHives = lib.attrNames ( lib.filterAttrs (_: hive: hive.certFingerprint != null) swarmCfg.hives ); in { options.services.hyperhive.swarm.hives = lib.mkOption { type = lib.types.attrsOf ( lib.types.submodule ( { name, ... }: { options = { domain = lib.mkOption { type = lib.types.str; # `.` is a derivation from two values an # operator had to state explicitly (both are required), not a # guess — and it is what makes this directory worth copying: # a conventional swarm is `{ pr1ma = { }; umbra = { }; }`, # names only, with a non-conventional hive saying so and only # that. A shared file is read far more often than written. # # Total rather than a throw when `swarm.domain` is unset, for # the reason ./hive-network.nix:155 gives in full: defaults # that interpolate the domain are forced *while the assertion # list evaluates*, so a throw here would replace the message # naming the missing option with a coercion error naming this # one. `.invalid` is reserved (RFC 2606) and fails loudly at # resolution if it ever escaped — which the required-domain # assertion is there to stop. default = if swarmCfg.domain == null then "${name}.invalid" else "${name}.${swarmCfg.domain}"; defaultText = lib.literalExpression ''"''${name}.''${services.hyperhive.swarm.domain}"''; example = "lab.example.com"; description = '' Public DNS domain this hive occupies — used for swarm-controller's hive roster, agent identity (qualified `agent@domain` labels), and Matrix federation discovery. Defaults to `.`, the convention every hive in a swarm follows, so a conventional directory is names only. Set it for a hive that is addressed by something else. ''; }; wireguardPublicKey = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "base64pubkey="; description = '' WireGuard public key for this hive's host. Required when `services.hyperhive.deploy.wireguard.enable = true` and you want this hive reachable over the mesh. Null = TLS- only peering (public internet, no mesh tunnel). ''; }; wireguardEndpoint = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "203.0.113.1:51820"; description = '' WireGuard endpoint for this hive in `host:port` form. Null = this hive has no reachable endpoint, so the tunnel is initiated from the other side. Reads like a fact about the relationship and is not: it says whether *this* hive can be dialled, which every other hive in the swarm needs the same answer to. ''; }; wireguardAddress = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "10.100.0.2/32"; description = '' IP address (with prefix) of this hive's host on the WireGuard mesh. Used as the `allowedIPs` for its WireGuard config entry (`./swarm-wireguard.nix`), so intra-swarm traffic can route over the mesh address rather than the public domain. Required to include a hive in the mesh (entries missing this field are silently excluded from `wg-hive`). ''; }; # Removed, and re-declared only so an existing definition # produces an error that says *what happened*. Without it the # module system says `The option ... does not exist`, which # tells an operator nothing about why it went or what replaced # it — and this field was set on real deployments. # # ⚠️ `lib.mkRemovedOptionModule` cannot do this job here, and # neither of its two halves survives inside a submodule: # its `apply = throw` fires only when the value is READ, and # nothing reads this (that being the point of removing # it); its `config.assertions` half would land on a submodule # that declares no `assertions` option at all. It is a # top-level tool. The assertion below is the working shape, and # it is the same one ./swarm-peers-removed.nix uses for the # analogous `peers..caCert`. certFingerprint = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; visible = false; internal = true; description = '' Removed. Trust inside a swarm comes from the secret store's pki root (docs/swarm/ca.md), which replaces per-hive leaf pinning entirely. ''; }; }; } ) ); default = { }; example = { pr1ma = { wireguardAddress = "10.100.0.1/32"; wireguardEndpoint = "203.0.113.1:51820"; }; edge = { domain = "edge.elsewhere.example"; wireguardAddress = "10.100.0.2/32"; }; }; description = '' Every hive in this swarm, keyed by `hiveName` — **including this host's own hive**. The same attrset is meant to be identical on every host in the swarm, so it can be written once and shared; `services.hyperhive.hiveName` is what makes a given host read it as "me and four others" rather than "five peers". Each entry needs no fields at all in the conventional case: a hive's `domain` defaults to `.`, so the whole directory is usually a list of names. It must contain an entry for `hiveName`, which is asserted — this host's own address is read out of it (it is where `services.hyperhive.domain` derives from), and a hive that lists everyone but itself would otherwise derive its own peer set as *everything* and peer with itself. ''; }; options.services.hyperhive.swarm.dashboardDomains = lib.mkOption { type = lib.types.attrsOf lib.types.str; readOnly = true; internal = true; description = '' Read-only: each hive's dashboard hostname, keyed by hive name, `.hives.`. Under the swarm domain so authelia's session cookie reaches it, whatever the hive's own domain is. Derived for every hive in `hives`, not only this one: the store's host writes each hive's leaf role from it, and authelia's host writes the operator-only rule for all of them. ''; }; options.services.hyperhive.swarm.dashboardDomain = lib.mkOption { type = lib.types.nullOr lib.types.str; readOnly = true; internal = true; description = '' Read-only: this hive's entry in `dashboardDomains`, or null where `hiveName` has no entry in `hives` (asserted below). ''; }; options.services.hyperhive.swarm.serviceDomains = lib.mkOption { type = lib.types.listOf lib.types.str; readOnly = true; internal = true; description = '' Read-only: the public hostnames of every swarm service, in a stable sorted order, identical on every host whichever services it runs. Derived once here because the CA that name-constrains these, the store role that issues for them and the leaf that carries them as SANs must agree exactly, and two modules each assembling the list is how they stop agreeing. Sorted and deduplicated deliberately: consumers compare this list against what they issued last time to decide whether to re-issue, so an unstable order would churn a certificate that other things are meant to pin. ''; }; options.services.hyperhive.swarm.localServiceDomains = lib.mkOption { type = lib.types.listOf lib.types.str; readOnly = true; internal = true; description = '' Read-only: the names in `serviceDomains` **this host actually fronts**, which is what its services leaf may carry. The swarm-wide set stays what the sub-CA is name-constrained to and what the store's issuing role permits; narrowing the leaf is what stops a private key on one gateway presenting a valid certificate for services that host does not serve and never has. Read off the rendered `services.nginx.virtualHosts` rather than off the per-service deploy flags: those flags are what the vhosts are already guarded on, so a second reading of them is a copy that drifts from the thing it claims to describe. Filtering `serviceDomains` rather than assembling a list beside it is that option's own constraint, and it applies here for the same reason. ''; }; config = { # Total on a null `swarm.domain` for the reason `hives..domain`'s # default gives. services.hyperhive.swarm.dashboardDomains = lib.mapAttrs ( name: _: if swarmCfg.domain == null then "${name}.hives.invalid" else "${name}.hives.${swarmCfg.domain}" ) swarmCfg.hives; services.hyperhive.swarm.dashboardDomain = if cfg.hiveName != null && swarmCfg.dashboardDomains ? ${cfg.hiveName} then swarmCfg.dashboardDomains.${cfg.hiveName} else null; services.hyperhive.swarm.serviceDomains = lib.sort (a: b: a < b) ( lib.unique (lib.filter (d: d != null && d != "") serviceDomains') ); services.hyperhive.swarm.localServiceDomains = lib.filter ( d: config.services.nginx.virtualHosts ? ${d} ) swarmCfg.serviceDomains; assertions = [ { # An EMPTY `hives` fires this too, deliberately: since # `swarm.domain` is required, every hive is in a swarm — a # swarm of one is still a swarm — so a directory with no entry # for this host is missing one either way. It also has to fire # here, because this host's own domain is now read out of the # directory: without the entry `services.hyperhive.domain` is # null and the generic required-domain assertion in # ./hive-network.nix would fire instead, naming an option the # operator should not be setting. # # Guarded on `hiveName != null` so the required-hiveName # assertion in ./hyperhive.nix is what fires for that case — # two assertions naming the same missing value is noise. assertion = cfg.hiveName == null || swarmCfg.hives ? ${cfg.hiveName}; message = '' services.hyperhive.swarm.hives has no entry for this hive (services.hyperhive.hiveName = "${toString cfg.hiveName}"). `hives` describes every hive in the swarm including this one, so that every host can share one identical attrset, and this hive's own domain is derived from its entry. Add: services.hyperhive.swarm.hives."${toString cfg.hiveName}" = { }; No fields are needed: `domain` defaults to `.`. Set it in the entry if this hive is addressed by something else. Declared hives: ${lib.concatStringsSep ", " (lib.attrNames swarmCfg.hives)} ''; } { # An error rather than a warning, deliberately. Re-declaring the # removed option above is what stops "option does not exist"; on # its own it would turn a config that sets the removed field into # one that quietly evaluates with the setting ignored, which is a # worse answer than an error. So this stays fatal until the field # is gone from the config. # # Wording follows `lib.mkRemovedOptionModule`'s, so it reads like # every other removal the module system reports. assertion = pinnedHives == [ ]; message = '' The option definition `services.hyperhive.swarm.hives..certFingerprint' no longer has any effect; please remove it. Still set on: ${lib.concatStringsSep ", " pinnedHives} It pinned a peer hive's TLS leaf for hive-c0re's own peer HTTPS checks, and was removed along with the dashboard feature it existed to serve — nothing else ever consumed it. There is no replacement, and none is needed for a hive inside this swarm: trust comes from the secret store's pki root (see docs/swarm/ca.md), which every hive chains to, so one anchor replaces per-hive pinning. What that genuinely drops is trusting a hive whose root this swarm does NOT own — another swarm's, or one keeping its own CA. That is a cross-swarm problem and wants a mechanism designed for it. ''; } (nameGuards.mustBeIdent { option = "services.hyperhive.swarm.hives"; names = hiveNames; }) (nameGuards.mustNotEqual { option = "services.hyperhive.swarm.hives"; names = hiveNames; reserved = reservedNames ++ certAuthCns; why = '' A hive's name is what other components address it by, and two of those uses resolve the clash silently rather than erroring. The collector names components `/` with the hive name as owner, and `//` merges them: the swarm tier's parts win, that hive's pipeline and `hive=` stamp disappear, and it keeps pushing into a route that goes nowhere. The secret store's cert-auth roles match on a certificate's common name, and a hive's own leaf carries its name. A hive named after a role's subject presents a certificate that role accepts — so it receives that principal's grants, which for the controller means write access to every hive's credentials and policies. Rename the hive. ''; }) (nameGuards.mustNotContain { option = "services.hyperhive.swarm.hives"; names = hiveNames; fragments = reservedFragments; why = '' Hive-scoped identifiers are composed from a hive name — `hive-`, `hive--agent` — so a name containing one of these produces an identifier that is also somebody else's. The failure is a wrong grant rather than an error: the client authenticates and receives another principal's permissions, and a NATS denial arrives as a timeout. Rename the hive. ''; }) ]; }; # `allSwarmServices` is declared in ./swarm-required-services.nix # together with the per-service `enable`s it asserts — it is a # deployment-shape switch rather than swarm bookkeeping, so it lives # with its consequences instead of here. options.services.hyperhive.swarm.snapshotStore = { address = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; example = "10.100.0.1"; description = '' Mesh address of the swarm's snapshot store — the single `btrfs receive` endpoint every hive in this swarm pushes agent snapshots to. Bare IP, no prefix. There is exactly **one** store per swarm, not one per peer: the receiver keys destinations by *agent*, so an agent that migrates between hives keeps a single unbroken incremental chain. Per-hive stores would split that chain in two, which is the case the store exists to serve. Null means this swarm has no store configured, and pushing fails saying so rather than guessing an address. Set it on every hive that pushes; the receiving host separately sets `services.hyperhive.snapshotStore.enable`. ''; }; port = lib.mkOption { type = lib.types.port; default = 51821; description = '' TCP port the swarm's snapshot store listens on. Must match the receiving host's `services.hyperhive.snapshotStore.port`. Defaulted (unlike `address`) because it is a shared convention both sides read from the same option docs — whereas an address is deployment-specific and cannot be guessed. ''; }; }; # How this hive reaches the swarm queue to offer its own status # (hive-c0re's `swarm_status`): the token endpoint below, which follows # the swarm's IdP address wherever that is, and the queue's URL under # `deploy.hive-controller`. The client secret is not an option: hive-c0re # reads it from the swarm's secret store under this hive's own # certificate (`deploy.bao.clientCertFile`). # # There is no `enable`: both coordinates set, plus a store identity, is # the enable. An extra flag would let a hive be configured-but-off, which # is one more state to explain and one more way to be silently quiet. options.services.hyperhive.swarm.statusPublish = { tokenEndpoint = lib.mkOption { type = lib.types.nullOr lib.types.str; default = "${swarmCfg.authelia.url}/api/oidc/token"; defaultText = lib.literalExpression ''"''${swarm.authelia.url}/api/oidc/token"''; example = "https://auth.example.com/api/oidc/token"; description = '' The swarm IdP's OAuth2 token endpoint. This hive mints a `client_credentials` access token there and presents it when connecting to the queue, which authenticates it as `hive-` — the client {file}`nix/host-modules/swarm-authelia.nix` already declares for every entry in {option}`services.hyperhive.swarm.hives`. ''; }; }; # What stays above is the one thing every hive shares: the address of the # swarm's token endpoint. The queue's URL is this machine's — where the # queue is *as seen from here* — so it hangs off the daemon that uses it. # `hive-controller` is the hive-c0re daemon's namespace, hence it lives # there rather than under a `deploy.statusPublish` of its own. options.services.hyperhive.deploy.hive-controller.statusPublish = { natsUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; default = "tls://${swarmCfg.nats.domain}:${toString swarmCfg.nats.port}"; defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"''; example = "tls://nats.example.com:4222"; description = '' Where the swarm queue listens, as seen from *this* hive. Defaults to the queue's name on every hive. The queue's own host resolves it locally, and any other hive through the operator's DNS. TLS only, and by name, since the queue's certificate carries the name and no address. A URL alone does not publish. This hive also needs its own store identity ({option}`services.hyperhive.deploy.bao.clientCertFile`), under which hive-c0re reads its `hive-` client secret from the swarm's secret store; `swarm-secret-publish`, on the host that runs authelia, puts it there. Without an identity, or with this set to null, this hive computes its own readiness as always and offers it to nobody, and the swarm controller reports it `never_reported`, which is the honest reading. ''; }; }; # The same queue, reached from one layer further in. An agent container has # its own network namespace and resolver, so this is its own option, even # though by name it defaults to the same URL as `statusPublish.natsUrl`. # # Only the address lives here. The credential does not: it is published per # hive and read out of the store by ./glue-queue-agent-credential.nix, which # owns `queue.agentCredentialDir` in the same namespace. options.services.hyperhive.deploy.hive-controller.queue.agentNatsUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; default = "tls://${swarmCfg.nats.domain}:${toString swarmCfg.nats.port}"; defaultText = lib.literalExpression ''"tls://''${swarm.nats.domain}:''${swarm.nats.port}"''; example = "tls://nats.example.com:4222"; description = '' Where the swarm queue listens, as an agent *container* on this host reaches it. Defaults to the queue's name on every hive. The agent resolves it through the bridge: on the queue's own host dnsmasq answers it with the bridge address, and elsewhere it forwards to the operator's DNS, as for the store's name. ⚠️ Never a loopback address: inside an agent's network namespace `127.0.0.1` is the agent, not this host, and the queue's certificate names no address anyway. Null means this hive's agents have not been given the queue's address. Together with {option}`services.hyperhive.swarm.statusPublish.tokenEndpoint` it is what decides whether the harness is handed queue coordinates at all. ''; }; }