add a semi-spread example deploy and a check that evaluates it

docs/examples/semi-spread/common.nix

@@ -0,0 +1,21 @@
+# Imported by every machine in the swarm: the swarm's shared facts, the VPN and
+# the names inside it. `services.hyperhive.swarm.*` has to be identical on all
+# of them, so it lives here and nowhere else.
+{
+  imports = [
+    ./wireguard.nix
+    ./dns.nix
+  ];
+
+  services.hyperhive.swarm = {
+    domain = "myswarm.example";
+    # Every hive, including the one a machine does not run. Names only: each
+    # hive's domain defaults to `<name>.myswarm.example`. A hive name may not
+    # contain "hive", "agent" or "swarm", so the machines `hive-one` and
+    # `hive-two` run the hives `one` and `two`.
+    hives = {
+      one = { };
+      two = { };
+    };
+  };
+}

docs/examples/semi-spread/dns.nix

@@ -0,0 +1,65 @@
+# Names inside the VPN. Every swarm name that is not public resolves to the
+# VPN address of the machine serving it, through a static table written on
+# each host. Identical on every host.
+#
+# Two readers need the table. Processes on the host read /etc/hosts
+# (`networking.hosts`). Containers (agents, swarm services) ask the hive's
+# dnsmasq, which ignores /etc/hosts, so it also reads the table through
+# `addn-hosts`.
+#
+# A host leaves out the names it serves itself: hyperhive already answers
+# those with the bridge address, and a table entry would override it.
+{
+  config,
+  lib,
+  ...
+}:
+let
+  mesh = import ./mesh.nix;
+  self = config.networking.hostName;
+  swarm = config.services.hyperhive.swarm;
+
+  # Which machine serves which name. A hive's machine is `hive-<name>` and it
+  # serves the hive's own names: its dashboard and its domain.
+  servedBy =
+    lib.mapAttrs' (
+      hive: entry:
+      lib.nameValuePair "hive-${hive}" [
+        swarm.dashboardDomains.${hive}
+        entry.domain
+      ]
+    ) swarm.hives
+    // {
+      swarm-central = [
+        swarm.domain
+        swarm.bao.domain
+        swarm.bao.ui.domain
+      ];
+      swarm-services = [
+        swarm.forge.domain
+        swarm.matrix.gatewayHost
+        swarm.authelia.domain
+        swarm.nats.domain
+        swarm.otel.domain
+        swarm.grafana.domain
+        swarm.victoriametrics.domain
+        swarm.victorialogs.domain
+      ];
+      swarm-heavyduty = [ swarm.attic.ownDomain ];
+    };
+
+  elsewhere = removeAttrs servedBy [ self ];
+in
+{
+  networking.hosts = lib.mapAttrs' (
+    machine: names: lib.nameValuePair mesh.${machine}.address names
+  ) elsewhere;
+
+  services.dnsmasq.settings.addn-hosts = "${builtins.toFile "swarm-vpn-hosts" (
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (
+        machine: names: "${mesh.${machine}.address} ${lib.concatStringsSep " " names}"
+      ) elsewhere
+    )
+  )}";
+}

docs/examples/semi-spread/hive.nix

@@ -0,0 +1,28 @@
+# A hive. Identical on hive-one and hive-two: the machine `hive-<name>` runs the
+# hive `<name>`.
+{ config, lib, ... }:
+let
+  # The leaves for this hive, copied from swarm-central, where the secret store
+  # runs: `client.pem` and `client-key.pem` from
+  # `/var/lib/swarm-bao-pki/hives/<name>/`, along with that directory's
+  # `hive-leaf.pem` and `hive-leaf-key.pem`, which the hive's gateway
+  # certificate comes from. Both pairs sit in one directory.
+  pki = "/var/lib/swarm-pki";
+in
+{
+  services.hyperhive.deploy = {
+    hive-controller = {
+      enable = true;
+      hiveName = lib.removePrefix "hive-" config.networking.hostName;
+    };
+    bao = {
+      clientCertFile = "${pki}/client.pem";
+      clientKeyFile = "${pki}/client-key.pem";
+    };
+  };
+
+  # The dashboard is reachable over the VPN only.
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [
+    config.services.hyperhive.gateway.httpsPort
+  ];
+}

docs/examples/semi-spread/mesh.nix

@@ -0,0 +1,33 @@
+# The VPN's facts: one entry per machine, read by every host (wireguard.nix,
+# dns.nix) and nothing else. Keyed by `networking.hostName`.
+#
+# `address` is the machine's address inside the VPN; `endpoint` is where the
+# other machines dial it; `publicKey` is the output of `wg pubkey` on that
+# machine. The keys below are placeholders: replace each with the real one.
+{
+  swarm-central = {
+    address = "10.77.0.1";
+    endpoint = "203.0.113.10:51820";
+    publicKey = "REPLACE-with-swarm-central-wg-pubkey";
+  };
+  swarm-services = {
+    address = "10.77.0.2";
+    endpoint = "203.0.113.11:51820";
+    publicKey = "REPLACE-with-swarm-services-wg-pubkey";
+  };
+  swarm-heavyduty = {
+    address = "10.77.0.3";
+    endpoint = "203.0.113.12:51820";
+    publicKey = "REPLACE-with-swarm-heavyduty-wg-pubkey";
+  };
+  hive-one = {
+    address = "10.77.0.11";
+    endpoint = "198.51.100.11:51820";
+    publicKey = "REPLACE-with-hive-one-wg-pubkey";
+  };
+  hive-two = {
+    address = "10.77.0.12";
+    endpoint = "198.51.100.12:51820";
+    publicKey = "REPLACE-with-hive-two-wg-pubkey";
+  };
+}

docs/examples/semi-spread/swarm-central.nix

@@ -0,0 +1,23 @@
+# swarm-central: the swarm controller (and the swarm UI it serves) and the
+# secret store.
+{ config, ... }:
+let
+  mesh = import ./mesh.nix;
+  hyperhive = config.services.hyperhive;
+in
+{
+  services.hyperhive = {
+    deploy.swarm-controller.enable = true;
+    deploy.bao.enable = true;
+
+    # The store also listens on this machine's VPN address, for the services
+    # and hives on other machines.
+    deploy.bao.extraListenAddresses = [ mesh.swarm-central.address ];
+
+    # The swarm UI is public.
+    gateway.openFirewall = true;
+  };
+
+  # The store is reachable over the VPN only.
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [ hyperhive.swarm.bao.port ];
+}

docs/examples/semi-spread/swarm-heavyduty.nix

@@ -0,0 +1,31 @@
+# swarm-heavyduty: the builder, the forge's CI runner and the binary cache.
+{ config, lib, ... }:
+let
+  hyperhive = config.services.hyperhive;
+
+  # Leaves copied from swarm-central, where the secret store runs. See
+  # swarm-services.nix for the helper's naming.
+  pki = "/var/lib/swarm-pki";
+  pair = option: stem: {
+    "${option}ClientCertFile" = "${pki}/${stem}.pem";
+    "${option}ClientKeyFile" = "${pki}/${stem}-key.pem";
+  };
+in
+{
+  services.hyperhive = {
+    deploy.builder.enable = true;
+    deploy.ci-runner.enable = true;
+    deploy.attic.enable = true;
+
+    deploy.bao = lib.mergeAttrsList [
+      (pair "builder" "builder")
+      (pair "ciRunner" "ci-runner")
+      (pair "attic" "attic")
+      (pair "forwarderOidc" "forwarder-oidc")
+      (pair "servicesIssuer" "services-issuer")
+    ];
+  };
+
+  # The cache is served through the gateway, over the VPN only.
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [ hyperhive.gateway.httpsPort ];
+}

docs/examples/semi-spread/swarm-services.nix

@@ -0,0 +1,64 @@
+# swarm-services: the forge, the matrix homeserver, SSO, the queue, and the
+# metrics and log stack.
+{ config, lib, ... }:
+let
+  hyperhive = config.services.hyperhive;
+
+  # The secret store runs on swarm-central, so each service here reads it with
+  # a leaf copied from there. `pair "grafanaOidc" "grafana-oidc"` names the
+  # options `deploy.bao.grafanaOidcClientCertFile` and `…KeyFile` and the files
+  # `grafana-oidc.pem` and `grafana-oidc-key.pem`.
+  pki = "/var/lib/swarm-pki";
+  pair = option: stem: {
+    "${option}ClientCertFile" = "${pki}/${stem}.pem";
+    "${option}ClientKeyFile" = "${pki}/${stem}-key.pem";
+  };
+in
+{
+  services.hyperhive = {
+    # Everything once-per-swarm that is not placed elsewhere below.
+    deploy.allSwarmServices = true;
+
+    # The forge and the matrix homeserver are public, and so is the SSO login
+    # they send browsers to.
+    gateway.openFirewall = true;
+
+    deploy.bao = lib.mergeAttrsList [
+      { enable = false; }
+      (pair "forgePublisher" "forge-publisher")
+      (pair "servicesIssuer" "services-issuer")
+      (pair "grafanaOidc" "grafana-oidc")
+      (pair "otelOidc" "otel-oidc")
+      (pair "forwarderOidc" "forwarder-oidc")
+      (pair "reviewClassifier" "review-classifier")
+      (pair "notifyRouter" "notify-router")
+      (pair "claudeLogin" "claude-login")
+      (pair "ciMetrics" "ci-metrics")
+    ];
+    deploy.matrix = {
+      ctlBaoClientCertFile = "${pki}/matrix-ctl.pem";
+      ctlBaoClientKeyFile = "${pki}/matrix-ctl-key.pem";
+    };
+    deploy.nats = {
+      baoClientCertFile = "${pki}/nats.pem";
+      baoClientKeyFile = "${pki}/nats-key.pem";
+      authBaoClientCertFile = "${pki}/nats-auth.pem";
+      authBaoClientKeyFile = "${pki}/nats-auth-key.pem";
+    };
+    deploy.swarm-secret-publisher = {
+      baoClientCertFile = "${pki}/secret-publisher.pem";
+      baoClientKeyFile = "${pki}/secret-publisher-key.pem";
+    };
+    deploy.victoriametrics = {
+      authBaoClientCertFile = "${pki}/metrics-auth.pem";
+      authBaoClientKeyFile = "${pki}/metrics-auth-key.pem";
+    };
+    deploy.victorialogs = {
+      authBaoClientCertFile = "${pki}/logs-auth.pem";
+      authBaoClientKeyFile = "${pki}/logs-auth-key.pem";
+    };
+  };
+
+  # The queue is reachable over the VPN only.
+  networking.firewall.interfaces.wg0.allowedTCPPorts = [ hyperhive.swarm.nats.port ];
+}

docs/examples/semi-spread/wireguard.nix

@@ -0,0 +1,24 @@
+# The VPN: plain NixOS WireGuard, one interface `wg0`, a full mesh built from
+# ./mesh.nix. Identical on every host; the host's own entry is picked by
+# `networking.hostName`.
+{ config, lib, ... }:
+let
+  mesh = import ./mesh.nix;
+  self = config.networking.hostName;
+in
+{
+  networking.wireguard.interfaces.wg0 = {
+    ips = [ "${mesh.${self}.address}/24" ];
+    listenPort = 51820;
+    # Created on the host by `wg genkey`, never in the store.
+    privateKeyFile = "/etc/wireguard/wg0.key";
+    peers = lib.mapAttrsToList (_: peer: {
+      inherit (peer) publicKey endpoint;
+      allowedIPs = [ "${peer.address}/32" ];
+      persistentKeepalive = 25;
+    }) (removeAttrs mesh [ self ]);
+  };
+
+  # The handshake arrives on the public interface.
+  networking.firewall.allowedUDPPorts = [ config.networking.wireguard.interfaces.wg0.listenPort ];
+}