hyperhive/nix/module-eval/swarm-services-switch.nix

# `checks.module-eval-swarm-services-switch` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
  shared,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    hiveWithController
    runGroup
    bridgePortsOrNone
    swarmServiceEnables
    ;

  # A dedicated services host with hives elsewhere: the swarm-services
  # toggle and no hive, the recipe docs/swarm/services.md gives.
  swarmServicesOnly = shared.servicesHere;

  # A services host with hives elsewhere and the store elsewhere too: no hive
  # and no store identity minted here.
  swarmServicesOnlyNoStore = swarmServicesBaoElsewhere;

  # A dedicated secret-store host: no hive, and the store's browser UI the
  # only swarm name its gateway fronts.
  storeOnlyNoHive = shared.baoOnly;

  # The homeserver on a host without a hive.
  matrixNoHive = hive {
    deploy.hive-controller.enable = false;
    deploy.matrix.enable = true;
  };

  # The same switch with every service it owns placed elsewhere, so the only
  # thing that can enable the gateway/resolver/bridge here is that toggle's
  # own module — each service module asserts them from its own `enable`.
  swarmServicesSwitchAlone = hive {
    deploy.hive-controller.enable = false;
    deploy.allSwarmServices = true;
    deploy.matrix.enable = false;
    deploy.authelia.enable = false;
    deploy.nats.enable = false;
    deploy.swarm-otel.enable = false;
    deploy.victoriametrics.enable = false;
    deploy.grafana.enable = false;
    deploy.victorialogs.enable = false;
    deploy.bao.enable = false;
    deploy.forgejo.enable = false;
  };

  # Ports asked for on such a host. The request is the whole condition: the
  # firewall hole exists because an operator named a port, not because the
  # hive is running, and an agent reaching a host service is a claim about
  # the host's own listeners either way.
  exposedPortsNoHive = hive {
    deploy.hive-controller.enable = false;
    network.exposeHostPorts = [ 5432 ];
  };

  # The swarm UI on the controller's host — the control the absence arm
  # needs, since nothing else in this suite renders this vhost and an arm
  # saying "it is not there" would hold just as well if it were never there.
  # Package stubbed per this file's header: the vhost roots at it. Set through
  # its old `deploy.swarm-ui.package` spelling, which must reach the vhost.
  #
  # Also sets both ignored UI switches to `false`: the vhost must render
  # regardless, and each must evaluate and warn. Neither
  # is read anywhere else, so they change nothing the other arms see.
  swarmUiHere = hive {
    deploy.swarm-controller.enable = true;
    deploy.swarm-ui.package = pkgs.emptyDirectory;
    deploy.swarm-ui.enable = false;
    swarm.ui.enable = false;
  };

  # The swarm's shared services hosted HERE without the all-local mode — a
  # services box with hives elsewhere, the shape ./host-modules/
  # swarm-required-services.nix documents the switch for.
  swarmServicesHere = shared.swarmServicesWithController;

  # Same, with the operator opting out of the classifier and the builder,
  # which the switch defaults on and everything else it owns stays on.
  swarmServicesOptedOut = hiveWithController {
    deploy.allSwarmServices = true;
    deploy.review-classifier.enable = false;
    deploy.builder.enable = false;
  };

  # Same, with the claude-code build the agents run pinned on this host. Any
  # package serves: only the path the containers are pointed at is read.
  swarmServicesPinnedClaude = hiveWithController {
    deploy.allSwarmServices = true;
    deploy.hive-controller.claudeCodePackage = pkgs.hello;
  };

  # Same, with one of those services placed on another host. Every derivation
  # in that module is `mkDefault` so this stays expressible.
  swarmServicesBaoElsewhere = hive {
    deploy.allSwarmServices = true;
    deploy.bao.enable = false;
  };

  # The swarm-wide forge objects are the controller's (forge/objects.rs), so
  # the mirror list and the org avatar go to its unit, not hive-c0re's.
  aMirror = {
    upstream = "https://example.invalid/tool";
    dest = "mirrors/tool";
  };
  controllerWithMirrors = hiveWithController {
    deploy.swarm-controller.enable = true;
    deploy.forgejo.enable = true;
    deploy.forgejo.mirrors = [ aMirror ];
    deploy.forgejo.actionsFromSelf = false;
    deploy.swarm-controller.configOrgAvatarPng = "/etc/marker-avatar.png";
  };
  mirrorsNoController = hive {
    deploy.forgejo.enable = true;
    deploy.forgejo.mirrors = [ aMirror ];
  };
  forgeOnly = shared.forgeHere;
  forgeWithController = hive {
    deploy.swarm-controller.enable = true;
    deploy.forgejo.enable = true;
  };
  controllerWithSelfActions = hive {
    deploy.swarm-controller.enable = true;
    deploy.forgejo.enable = true;
    deploy.forgejo.mirrors = [ aMirror ];
    deploy.forgejo.actionsFromSelf = true;
  };

  # The controller with every shared service on another host, and the same
  # controller hosting them all: the swarm UI's links must not tell the two
  # apart.
  inherit (shared) controllerAlone;
  controllerWithServices = hive {
    deploy.swarm-controller.enable = true;
    deploy.allSwarmServices = true;
  };

  # Each shared service with a web UI, and the link it must have: the label
  # and the swarm-level domain the URL is built from.
  swarmServiceLinkDomains =
    let
      s = controllerAlone.services.hyperhive.swarm;
    in
    {
      Authelia = s.authelia.domain;
      Grafana = s.grafana.domain;
      Metrics = s.victoriametrics.domain;
      Logs = s.victorialogs.domain;
      Matrix = s.matrix.gatewayHost;
      Bao = s.bao.ui.domain;
      Forge = s.forge.domain;
    };
  swarmServiceLinksOf =
    cfg:
    lib.filter (l: swarmServiceLinkDomains ? ${l.label}) cfg.services.hyperhive.swarm.controller.links;

  # Every service container at once: the services-only host plus the CI
  # runner. The CI runner, the review classifier, the builder, the claude login
  # minter, the metrics store, the homeserver and authelia are the seven containers on a
  # netns of their own.
  serviceContainersWithCi = hive {
    deploy.hive-controller.enable = false;
    deploy.allSwarmServices = true;
    deploy.ci-runner.enable = true;
  };
  serviceContainers = serviceContainersWithCi.containers;

  # The metrics store on a build older than its socket needs. Only its
  # `assertions` are read, and the package is a stub carrying a version.
  oldMetricsStore = hive {
    deploy.victoriametrics.enable = true;
    deploy.victoriametrics.package = pkgs.emptyDirectory // {
      version = "1.152.0";
    };
  };
  # Failing assertions first, then their messages: a message is only
  # well-defined when its assertion fails (nixpkgs' fileSystems one throws
  # otherwise).
  failedMetricsStoreVersion =
    host:
    lib.filter (lib.hasInfix "services.hyperhive.deploy.victoriametrics.package must be") (
      map (a: a.message) (lib.filter (a: !a.assertion) host.assertions)
    );
  cases = [
    {
      # `lib.all` over an empty set holds vacuously, so the roster is counted
      # before it is read: a helper that lost a member would otherwise turn
      # this case green by measuring nothing.
      name = "hosting the swarm's shared services turns on every service that switch owns";
      ok =
        let
          es = swarmServiceEnables swarmServicesHere;
        in
        lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es);
    }
    {
      # The container is the control that the toggle reaches the module, not
      # only the option; `nats` is the control that the switch is on at all.
      name = "hosting the swarm's shared services turns on the review classifier";
      ok =
        swarmServicesHere.services.hyperhive.deploy.review-classifier.enable
        && swarmServicesHere.containers ? swarm-review-classifier
        && swarmServicesHere.services.hyperhive.deploy.nats.enable;
    }
    {
      # The default yields to an operator who has spoken. The same fixture
      # without the line is the control.
      name = "an explicit false keeps the review classifier off where the shared services are hosted";
      ok =
        !swarmServicesOptedOut.services.hyperhive.deploy.review-classifier.enable
        && !(swarmServicesOptedOut.containers ? swarm-review-classifier)
        && swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
    }
    {
      # Same as the classifier's case above, for the builder: the opt-out is
      # the only difference from `swarmServicesHere`, which has the container.
      name = "an explicit false keeps the swarm builder off where the shared services are hosted";
      ok =
        !swarmServicesOptedOut.services.hyperhive.deploy.builder.enable
        && !(swarmServicesOptedOut.containers ? swarm-builder)
        && swarmServicesHere.containers ? swarm-builder
        && swarmServicesOptedOut.services.hyperhive.deploy.nats.enable;
    }
    {
      # The classifier's module asserts a store identity that only the store's
      # host mints, so with the store placed elsewhere the default stays off
      # rather than failing that host's eval. `nats` is the control.
      name = "the review classifier stays off where the shared services are hosted without the store";
      ok =
        !swarmServicesBaoElsewhere.services.hyperhive.deploy.review-classifier.enable
        && !(swarmServicesBaoElsewhere.containers ? swarm-review-classifier)
        && swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
    }
    {
      # The builder's module asserts its own store identity, so the same
      # holds for it. `nats` is the control.
      name = "the swarm builder stays off where the shared services are hosted without the store";
      ok =
        !swarmServicesBaoElsewhere.services.hyperhive.deploy.builder.enable
        && !(swarmServicesBaoElsewhere.containers ? swarm-builder)
        && swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
    }
    {
      # Static, not `DynamicUser`: the user is declared in the container and
      # the unit names it. `nats` is the control that the switch is on at all.
      name = "hosting the swarm's shared services runs the claude login minter in its container as a static user";
      ok =
        let
          c = swarmServicesHere.containers.swarm-claude-login.config;
          sc = c.systemd.services.swarm-claude-login.serviceConfig;
        in
        swarmServicesHere.services.hyperhive.deploy.claude-login.enable
        && c.users.users.swarm-claude-login.isSystemUser
        && sc.User == "swarm-claude-login"
        && !(sc.DynamicUser or false)
        && swarmServicesHere.services.hyperhive.deploy.nats.enable;
    }
    {
      # The minter's module asserts the store identity the store's host mints,
      # as the classifier's does.
      name = "the claude login minter stays off where the shared services are hosted without the store";
      ok =
        !swarmServicesBaoElsewhere.services.hyperhive.deploy.claude-login.enable
        && !(swarmServicesBaoElsewhere.containers ? swarm-claude-login)
        && swarmServicesBaoElsewhere.services.hyperhive.deploy.nats.enable;
    }
    {
      # The CLI is the one the agents run, not the container's own nixpkgs'.
      # Unpinned is the control: it falls back to the container's own `claude-code`.
      # serve.json is what hive-c0re hands every agent's `HYPERHIVE_CLAUDE_CODE_PATH` from.
      name = "the agents, the claude login minter and the review classifier run the claude-code this host pins";
      ok =
        let
          claudeOf =
            host: container: unit: var:
            host.containers.${container}.config.systemd.services.${unit}.environment.${var};
          pinned = "${pkgs.hello}/bin/claude";
          servedClaude =
            host:
            (builtins.fromJSON (
              builtins.unsafeDiscardStringContext host.environment.etc."hyperhive/serve.json".text
            )).claude_code_path;
        in
        servedClaude swarmServicesPinnedClaude == "${pkgs.hello}"
        && servedClaude swarmServicesHere == null
        &&
          claudeOf swarmServicesPinnedClaude "swarm-claude-login" "swarm-claude-login"
            "SWARM_CLAUDE_LOGIN_CLAUDE" == pinned
        &&
          claudeOf swarmServicesPinnedClaude "swarm-review-classifier" "swarm-review-classifier"
            "SWARM_REVIEW_CLASSIFIER_CLAUDE" == pinned
        && lib.hasInfix "claude-code" (
          claudeOf swarmServicesHere "swarm-claude-login" "swarm-claude-login" "SWARM_CLAUDE_LOGIN_CLAUDE"
        )
        && lib.hasInfix "claude-code" (
          claudeOf swarmServicesHere "swarm-review-classifier" "swarm-review-classifier"
            "SWARM_REVIEW_CLASSIFIER_CLAUDE"
        );
    }
    {
      # The switch fills in for an operator who has not spoken and yields to
      # one who has — that is what keeps a shared service placeable on a host
      # of its own. A plain assignment or `mkForce` would satisfy both cases
      # above and break this one. `nats` is the control: without it the case
      # also passes on a fixture where nothing came on at all.
      name = "placing one shared service elsewhere survives the switch that would enable it";
      ok =
        let
          es = swarmServiceEnables swarmServicesBaoElsewhere;
        in
        !es.bao && es.nats;
    }
    {
      # The controller sits on the OTHER tier: `singleHostSwarm` places it
      # (./host-modules/local-defaults.nix) and swarm-ui follows the
      # controller. Pinned so that moving a service between tiers is a
      # decision someone makes rather than a merge nobody reads.
      name = "hosting the swarm's shared services does not make a hive the swarm's control plane";
      ok =
        !swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable
        && !(swarmServicesHere.services.nginx.virtualHosts ? "t.local");
    }
    {
      # Control for the arm above, and the one place this slice is not inert:
      # the ports are opened because they were named, on a host that never
      # turned the hive on. The bridge firewall is the host's own, so there is
      # nothing here for the hive toggle to have been protecting.
      name = "a named exposeHostPorts opens its bridge port on the host's own say-so";
      ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive);
    }
    {
      name = "the old swarm UI package spelling sets the controller's UI package, which the vhost serves";
      ok =
        let
          uiPackage = swarmUiHere.services.hyperhive.deploy.swarm-controller.ui.package;
        in
        uiPackage == pkgs.emptyDirectory
        && swarmUiHere.services.nginx.virtualHosts."t.local".locations."/".root == "${uiPackage}"
        && lib.any (
          w:
          lib.hasInfix "services.hyperhive.deploy.swarm-ui.package" w
          && lib.hasInfix "services.hyperhive.deploy.swarm-controller.ui.package" w
        ) swarmUiHere.warnings;
    }
    {
      # Control: `swarmServicesHere` sets neither ignored switch.
      name = "a definition of an ignored swarm UI switch evaluates and warns, naming the option and the controller";
      ok =
        let
          warned =
            m: path:
            lib.any (
              w:
              lib.hasInfix "services.hyperhive.${path} no longer has any effect" w
              && lib.hasInfix "swarm-ui now runs wherever swarm-controller runs" w
            ) m.warnings;
          paths = [
            "deploy.swarm-ui.enable"
            "swarm.ui.enable"
          ];
        in
        lib.all (warned swarmUiHere) paths && !(lib.any (warned swarmServicesHere) paths);
    }
    {
      # The swarm-services toggle enables them explicitly, from its own
      # module rather than from any of their defaults.
      name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself";
      ok =
        swarmServicesSwitchAlone.services.hyperhive.gateway.enable
        && swarmServicesSwitchAlone.services.hyperhive.gateway.dns.enable
        && swarmServicesSwitchAlone.services.hyperhive.network.enable;
    }
    {
      # No hive, so no hive leaf, no hive identity and no hive-domain vhost:
      # the default certificate is the swarm-services leaf, or a throwaway
      # pair until it lands. A host fronting many swarm names waits for that
      # leaf; a store-only host does not. The control is a hive host with
      # the same services.
      name = "a gateway host without a hive serves the swarm-services leaf as its default cert and has no hive-leaf unit";
      ok =
        let
          # Every hive-leaf reference a unit could carry: the leaf files,
          # the identity's file and subject. The store's own per-hive role
          # unit is left out: it writes roles for the hives in the swarm
          # directory, which live elsewhere.
          hiveLeafRefs = [
            "gateway.pem"
            "gateway-key.pem"
            "hive-leaf"
            "swarm-bao-hive-leaf-"
          ];
          noHiveLeafAnywhere =
            m:
            let
              units = removeAttrs m.systemd.services [ "swarm-bao-hive-leaf-policy" ];
              texts = lib.concatMap (u: [
                (u.script or "")
                (builtins.toJSON (u.environment or { }))
              ]) (lib.attrValues units);
            in
            !(lib.any (t: lib.any (r: lib.hasInfix r t) hiveLeafRefs) texts)
            && m.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile == null
            && noHiveSurface m;
          # No hive-domain or dashboard vhost, and no vhost on the host carrying the
          # hive's locations: c0re's `/health/`, the agent routes, swagger,
          # or the hive domain's matrix discovery and `/matrix/` redirect.
          noHiveSurface =
            m:
            let
              vh = m.services.nginx.virtualHosts;
            in
            !(vh ? "h1.t.local")
            && !(vh ? "h1.hives.t.local")
            && !(lib.any (
              v:
              lib.any (l: (v.locations or { }) ? ${l}) [
                "/health/"
                "/agent/"
                "/api/docs/"
                "/matrix/"
                "= /.well-known/matrix/client"
                "= /.well-known/matrix/server"
              ]
              || lib.hasInfix "agents.conf" (v.extraConfig or "")
            ) (lib.attrValues vh));
          noHive =
            m:
            let
              s = m.systemd.services;
            in
            !(s ? hive-tls-resign) && !(m.systemd.timers ? hive-tls-resign) && noHiveLeafAnywhere m;
          waitsForLeaf =
            m:
            let
              s = m.systemd.services;
            in
            lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.requiredBy
            && lib.elem "hive-gateway-self-signed-cert.service" s.swarm-services-cert.before;
          # The cert it installs on the default and swarm-services paths:
          # the services leaf when present.
          importsServicesLeaf =
            m:
            let
              imp = m.systemd.services.hive-gateway-self-signed-cert;
            in
            lib.hasInfix "cert=/var/lib/hive-tls/swarm-services.pem" imp.script
            && lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/cert.pem'' imp.script
            && lib.hasInfix ''install -m 0644 "$cert" /var/lib/hive-gateway/tls/swarm-services.pem'' imp.script;
          # Without it: a throwaway pair where the gateway does not wait for
          # the leaf, a failure naming the issuing unit where it does.
          standsInThrowaway =
            m:
            let
              imp = m.systemd.services.hive-gateway-self-signed-cert;
            in
            lib.hasInfix "openssl req -x509" imp.script
            && lib.hasInfix "cert=/var/lib/hive-gateway/tls/fallback.pem" imp.script
            && lib.any (p: lib.hasPrefix "openssl" (p.pname or (p.name or ""))) imp.path;
          failsWithoutLeaf =
            m:
            let
              s = m.systemd.services.hive-gateway-self-signed-cert.script;
            in
            lib.hasInfix "swarm-services-cert.service issues it" s
            && lib.hasInfix "exit 1" s
            && !(lib.hasInfix "fallback.pem" s);
          store = storeOnlyNoHive;
          hs = swarmServicesHere.systemd.services;
        in
        noHive swarmServicesOnly
        && noHive swarmServicesOnlyNoStore
        && noHive store
        && importsServicesLeaf swarmServicesOnly
        && importsServicesLeaf swarmServicesOnlyNoStore
        && importsServicesLeaf store
        # Fronting many swarm names: the gateway waits for their leaf, and
        # its import fails without it.
        && waitsForLeaf swarmServicesOnly
        && waitsForLeaf swarmServicesOnlyNoStore
        && failsWithoutLeaf swarmServicesOnly
        && failsWithoutLeaf swarmServicesOnlyNoStore
        && standsInThrowaway store
        && !(failsWithoutLeaf store)
        # The store's UI alone: its nginx carries the passthrough to the
        # store, so it starts on the fallback with the store sealed.
        &&
          store.services.hyperhive.deploy.hive-controller.tls.localServiceDomains
          == [ store.services.hyperhive.swarm.bao.ui.domain ]
        && !(waitsForLeaf store)
        # The homeserver without a hive: no hive-domain vhost at all, and the
        # homeserver's own vhost still proxies `/_matrix/`.
        && noHive matrixNoHive
        && (
          let
            gh = matrixNoHive.services.hyperhive.swarm.matrix.gatewayHost;
            vh = matrixNoHive.services.nginx.virtualHosts;
          in
          gh != null && vh ? ${gh} && lib.any (lib.hasPrefix "/_matrix") (lib.attrNames vh.${gh}.locations)
        )
        && !(store.services.nginx.virtualHosts ? "h1.t.local")
        && lib.hasInfix "want_svc=1" store.systemd.services.swarm-services-cert.script
        # Control: a hive host with the same services keeps the hive leaf,
        # its unit and identity, the dashboard's vhost and the hive-domain one.
        && hs ? hive-tls-resign
        && lib.hasInfix "install -m 0644 /var/lib/hive-tls/gateway.pem /var/lib/hive-gateway/tls/cert.pem" hs.hive-gateway-self-signed-cert.script
        && swarmServicesHere.services.hyperhive.deploy.hive-controller.tls.baoHiveIdentityCertFile != null
        && (
          let
            vh = swarmServicesHere.services.nginx.virtualHosts;
            l = vh."h1.hives.t.local".locations;
          in
          l ? "/api/" && l ? "/health/" && vh."h1.t.local".locations ? "= /.well-known/matrix/client"
        )
        && !(noHiveLeafAnywhere swarmServicesHere);
    }
    {
      # Each service is gated on its own deployment toggle alone, so a host
      # that runs no hive still runs every shared service the switch turns
      # on, plus the plumbing and the self-signed CA they are served through.
      # Nothing starts the hive's own daemons. `hive-c0re` is checked by what
      # starts it rather than by key: ../host-modules/hive-network.nix's bridge
      # block writes to its environment wherever the bridge is up.
      name = "a services-only host runs the swarm's shared services and no hive";
      ok =
        let
          c = swarmServicesOnly.containers;
          s = swarmServicesOnly.systemd.services;
          h = swarmServicesOnly.services.hyperhive;
        in
        lib.all (m: c ? ${m}) [
          "swarm-authelia"
          "swarm-bao"
          "swarm-grafana"
          "swarm-victorialogs"
          "swarm-victoriametrics"
          "swarm-otel"
          "swarm-nats"
          "hive-matrix"
          "hive-forge"
        ]
        && h.gateway.enable
        && h.gateway.dns.enable
        && h.network.enable
        && s ? hive-tls-ca
        && s ? swarm-bao-pki
        && (s.hive-c0re.wantedBy or [ ]) == [ ]
        && !(swarmServicesOnly.systemd.sockets ? hive-c0re)
        && !(swarmServicesOnly.systemd.sockets ? hive-priv)
        && !(s ? swarm-bao-queue-agent);
    }
    {
      # On a controller that runs none of them, so the link cannot come from
      # the service's own module.
      name = "the swarm UI links every shared service with a web UI at its swarm domain";
      ok =
        lib.listToAttrs (map (l: lib.nameValuePair l.label l.url) (swarmServiceLinksOf controllerAlone))
        == lib.mapAttrs (_: domain: "https://${domain}/") swarmServiceLinkDomains;
    }
    {
      # Exact list equality also catches an entry rendered twice where the
      # service runs on the controller's own host.
      name = "the swarm UI's service links are the same whichever host runs the services";
      ok =
        lib.length (swarmServiceLinksOf controllerAlone)
        == lib.length (lib.attrNames swarmServiceLinkDomains)
        && swarmServiceLinksOf controllerWithServices == swarmServiceLinksOf controllerAlone;
    }
    {
      # The in-container option drives the container's firewall, and host
      # `false` with container `true` would let the container's
      # firewall.service rewrite the HOST ruleset. Counted first so a fixture
      # that lost a container can't pass by comparing fewer of them.
      name = "every service container restates its host-side privateNetwork";
      ok =
        lib.length (lib.attrNames serviceContainers) == 13
        && lib.all (
          c: c.privateNetwork == (c.config.services.hyperhive.swarmContainer.privateNetwork or null)
        ) (lib.attrValues serviceContainers);
    }
    {
      # Both values come from ../container-modules/swarm-container.nix.
      name = "a service container on the host netns runs no firewall or resolvconf of its own";
      ok =
        let
          shared = lib.filter (c: !c.privateNetwork) (lib.attrValues serviceContainers);
        in
        lib.length shared == 6
        && lib.all (
          c: !c.config.networking.firewall.enable && !c.config.networking.resolvconf.enable
        ) shared;
    }
    {
      # The `privateNetwork = true` and `writesOwnResolvConf = false` arms:
      # NixOS enables both by default, so the module must leave them alone.
      name = "a service container on a netns of its own keeps its firewall and resolvconf";
      ok =
        let
          c = serviceContainers.swarm-ci-runner.config;
        in
        c.networking.firewall.enable && c.networking.resolvconf.enable;
    }
    {
      # No interface but loopback, so the socket is the store's only listener
      # the host can reach. Every location that forwards to the store goes
      # through it; the internal auth subrequests are the only other upstreams.
      # The read, push and scrape locations are named so an empty filter
      # can't pass.
      name = "the metrics store runs on a netns of its own and the gateway reaches it over its socket";
      ok =
        let
          host = serviceContainersWithCi;
          c = host.containers.swarm-victoriametrics;
          dir = "/run/swarm-victoriametrics";
          sock = "${dir}/victoriametrics.sock";
          viaSock = path: "http://unix:${sock}:${path}";
          locs =
            host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.victoriametrics.domain}.locations;
          toStore = map (l: l.proxyPass) (
            lib.filter (l: l.proxyPass != null && !(lib.hasInfix "internal;" l.extraConfig)) (
              lib.attrValues locs
            )
          );
          unit = c.config.systemd.services.victoriametrics;
          nginxGid = host.ids.gids.nginx;
        in
        c.privateNetwork
        && c.hostBridge == null
        && c.hostAddress == null
        && c.config.services.victoriametrics.listenAddress == "unix:${sock}"
        && locs."= /api/v1/query".proxyPass == viaSock "/api/v1/query"
        && locs."= /opentelemetry/api/v1/push".proxyPass == viaSock "/opentelemetry/api/v1/push"
        && locs."= /metrics".proxyPass == viaSock "/metrics"
        && lib.all (lib.hasPrefix (viaSock "")) toStore
        && c.bindMounts.${dir}.hostPath == dir
        && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
        && c.config.users.groups.gateway-nginx.gid == nginxGid
        && unit.serviceConfig.Group == "gateway-nginx"
        && !unit.serviceConfig.DynamicUser
        && unit.serviceConfig.UMask == "0007"
        && lib.hasInfix "--unix-socket ${sock}" unit.postStart;
    }
    {
      # On the bridge by DHCP, like the CI runner, and its socket the only
      # listener the gateway dials. `address` unset is what keeps tuwunel off
      # TCP. Every vhost location that forwards goes through the socket; the
      # `/_matrix/` one is named so an empty filter can't pass.
      name = "the homeserver runs on a netns of its own and the gateway reaches it over its socket";
      ok =
        let
          host = serviceContainersWithCi;
          net = host.services.hyperhive.network;
          c = host.containers.hive-matrix;
          dir = "/run/hive-matrix";
          sock = "${dir}/tuwunel.sock";
          locs =
            host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.matrix.gatewayHost}.locations;
          forwarded = lib.filter (p: p != null) (map (l: l.proxyPass) (lib.attrValues locs));
          tuwunel = c.config.services.matrix-tuwunel;
          unit = c.config.systemd.services.tuwunel;
          nginxGid = host.ids.gids.nginx;
        in
        c.privateNetwork
        && c.hostBridge == net.bridgeName
        && c.hostAddress == null
        && c.localAddress == null
        && c.config.networking.interfaces.eth0.useDHCP
        && c.config.networking.nameservers == [ net.bridgeIp ]
        && c.config.networking.firewall.enable
        && c.config.networking.resolvconf.enable
        && (tuwunel.settings.global.address or null) == null
        && tuwunel.settings.global.unix_socket_path == sock
        && tuwunel.settings.global.unix_socket_perms == 660
        && locs."/_matrix/".proxyPass == "http://unix:${sock}:"
        && forwarded == [ "http://unix:${sock}:" ]
        && c.bindMounts.${dir}.hostPath == dir
        && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
        && c.config.users.groups.gateway-nginx.gid == nginxGid
        && tuwunel.group == "gateway-nginx"
        && unit.serviceConfig.Group == "gateway-nginx"
        && lib.elem dir unit.serviceConfig.ReadWritePaths;
    }
    {
      # On the bridge by DHCP, with all three listeners on sockets: nothing
      # the bridge can dial. Every vhost location that forwards goes through
      # one of the sockets, except the internal authz subrequest, which is
      # `https://`. The main, bridge and metrics locations are named so an
      # empty filter can't pass.
      name = "authelia runs on a netns of its own and the gateway reaches all three listeners over sockets";
      ok =
        let
          host = serviceContainersWithCi;
          net = host.services.hyperhive.network;
          c = host.containers.swarm-authelia;
          dir = "/run/swarm-authelia";
          sock = n: "${dir}/${n}.sock";
          locs = host.services.nginx.virtualHosts.${host.services.hyperhive.swarm.authelia.domain}.locations;
          forwarded = lib.filter (p: p != null && !(lib.hasPrefix "https://" p)) (
            map (l: l.proxyPass) (lib.attrValues locs)
          );
          inner = c.config;
          settings = inner.services.authelia.instances.swarm.settings;
          unit = inner.systemd.services.authelia-swarm;
          bridge = inner.systemd.services.swarm-authelia-bridge;
          nginxGid = host.ids.gids.nginx;
        in
        c.privateNetwork
        && c.hostBridge == net.bridgeName
        && c.hostAddress == null
        && c.localAddress == null
        && inner.networking.interfaces.eth0.useDHCP
        && inner.networking.firewall.enable
        && settings.server.address == "unix://${sock "authelia"}?umask=0007"
        && settings.telemetry.metrics.address == "unix://${sock "metrics"}?umask=0007"
        && bridge.environment.SWARM_AUTHELIA_BRIDGE_SOCKET == sock "bridge"
        && !(bridge.environment ? SWARM_AUTHELIA_BRIDGE_BIND)
        && locs."/".proxyPass == "http://unix:${sock "authelia"}:/"
        && locs."/bridge/".proxyPass == "http://unix:${sock "bridge"}:/"
        && locs."= /metrics".proxyPass == "http://unix:${sock "metrics"}:/"
        && lib.all (lib.hasPrefix "http://unix:${dir}/") forwarded
        && c.bindMounts.${dir}.hostPath == dir
        && builtins.elem "d ${dir} 0770 root ${toString nginxGid} - -" host.systemd.tmpfiles.rules
        && inner.users.groups.gateway-nginx.gid == nginxGid
        && unit.serviceConfig.Group == "gateway-nginx"
        && bridge.serviceConfig.Group == "gateway-nginx"
        && lib.elem dir unit.serviceConfig.ReadWritePaths;
    }
    {
      # The DynamicUser layout keeps the history under /var/lib/private; the
      # static user's StateDirectory is the plain path. The move runs as its
      # own unit ahead of the store.
      name = "the metrics store's history moves from the DynamicUser path before the store starts";
      ok =
        let
          cfg = serviceContainers.swarm-victoriametrics.config;
          m = cfg.systemd.services.victoriametrics-state-migrate;
        in
        cfg.services.victoriametrics.stateDir == "victoriametrics"
        && builtins.elem "victoriametrics.service" m.before
        && builtins.elem "victoriametrics.service" m.requiredBy
        && lib.hasInfix "old=/var/lib/private/victoriametrics" m.script
        && lib.hasInfix "new=/var/lib/victoriametrics" m.script
        && lib.hasInfix ''mv "$old" "$new"'' m.script
        && lib.hasInfix ''chown -R victoriametrics:gateway-nginx "$new"'' m.script;
    }
    {
      # The control is the default package on the all-services host.
      name = "a metrics store package older than 1.153.0 fails evaluation";
      ok =
        let
          old = failedMetricsStoreVersion oldMetricsStore;
        in
        lib.length old == 1
        && lib.hasInfix "1.152.0" (lib.head old)
        && failedMetricsStoreVersion serviceContainersWithCi == [ ];
    }
    {
      name = "a service container that sets no stateVersion of its own is on 26.05";
      ok = serviceContainers.swarm-victoriametrics.config.system.stateVersion == "26.05";
    }
    {
      name = "a service container that pins its own stateVersion keeps it";
      ok = serviceContainers.hive-forge.config.system.stateVersion == "25.11";
    }
    {
      # An operator's explicit `false` beats every `mkDefault` assertion,
      # which is what keeps "asserted by whoever needs it" from being a
      # setting the operator cannot turn off.
      name = "an explicit gateway.enable = false wins over the modules asserting it";
      ok = !(hiveWithController { gateway.enable = false; }).services.nginx.enable;
    }
    {
      name = "declared forge mirrors reach the swarm-controller, not hive-c0re";
      ok =
        builtins.fromJSON controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS
        == [ aMirror ]
        && !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HYPERHIVE_FORGE_MIRRORS);
    }
    {
      name = "forge mirrors on a host without the controller warn that nothing seeds them";
      ok =
        let
          warns = cfg: lib.any (lib.hasInfix "declare them on the swarm-controller's host") cfg.warnings;
        in
        warns mirrorsNoController && !(warns controllerWithMirrors);
    }
    {
      # The mirrors are what `self` resolves `actions/*` to, so the two come
      # together: one per action `.forgejo/workflows/` names. The control is
      # the same forge without the option.
      name = "actionsFromSelf adds the actions mirrors and resolves actions from the forge";
      ok =
        let
          mirrorsOf =
            m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
          actionsUrlOf =
            m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
        in
        map (m: m.dest) (mirrorsOf controllerWithSelfActions) == [
          "mirrors/tool"
          "actions/checkout"
          "actions/upload-artifact"
        ]
        && actionsUrlOf controllerWithSelfActions == "self"
        && mirrorsOf controllerWithMirrors == [ aMirror ]
        && actionsUrlOf controllerWithMirrors == null;
    }
    {
      # The default follows the controller, the only thing that seeds the
      # checkout mirror: a forge alone must not resolve actions from itself.
      name = "actionsFromSelf defaults to whether this host runs the swarm-controller";
      ok =
        let
          actionsUrlOf =
            m: m.containers.hive-forge.config.services.forgejo.settings.actions.DEFAULT_ACTIONS_URL or null;
          mirrorsOf =
            m: builtins.fromJSON m.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_FORGE_MIRRORS;
        in
        !forgeOnly.services.hyperhive.deploy.forgejo.actionsFromSelf
        && actionsUrlOf forgeOnly == null
        && forgeOnly.warnings == [ ]
        && !(
          (forgeOnly.systemd.services.swarm-controller.environment or { }) ? SWARM_CONTROLLER_FORGE_MIRRORS
        )
        && !(
          (mirrorsNoController.systemd.services.swarm-controller.environment or { })
            ? SWARM_CONTROLLER_FORGE_MIRRORS
        )
        && forgeWithController.services.hyperhive.deploy.forgejo.actionsFromSelf
        && actionsUrlOf forgeWithController == "self"
        && lib.elem "actions/checkout" (map (m: m.dest) (mirrorsOf forgeWithController));
    }
    {
      name = "the org avatar lands on the controller's unit, not hive-c0re's";
      ok =
        controllerWithMirrors.systemd.services.swarm-controller.environment.SWARM_CONTROLLER_CONFIG_ORG_AVATAR_PNG
        == "/etc/marker-avatar.png"
        && !(controllerWithMirrors.systemd.services.hive-c0re.environment ? HIVE_ORG_AVATAR_PNG);
    }
    {
      # Where the store runs here, each reader waits for its container and
      # the PKI unit minting the leaf it logs in with, and wants both. It
      # never requires the PKI unit: a refused pki start fails a requirer's
      # job with `dependency`, which its `Restart=on-failure` never retries.
      # Elsewhere neither unit exists. On the store's host without the
      # readers, none of them is rendered as a stub. Rosters are counted: an
      # empty one holds vacuously.
      name = "a store reader waits for and wants the store's container and PKI unit only where the store runs here, and requires neither";
      ok =
        let
          machine = serviceContainersWithCi.services.hyperhive.swarm.bao.machine;
          storeUnits = [
            "swarm-bao-pki.service"
            "container@${machine}.service"
          ];
          readers = [
            "swarm-bao-forge-oidc"
            "swarm-bao-matrix-oidc"
            "swarm-bao-hive-otel-oidc"
            "swarm-bao-grafana-oidc"
            "swarm-bao-grafana-admin"
            "swarm-bao-grafana-metrics-bearer"
            "swarm-bao-grafana-controller-token"
            "swarm-bao-nats-callout-xkey"
            "swarm-bao-ci-runner"
            "swarm-bao-otel-oidc"
          ];
          here = serviceContainersWithCi.systemd.services;
          elsewhere = swarmServicesBaoElsewhere.systemd.services;
          readersHere = lib.filter (u: here ? ${u}) readers;
          readersElsewhere = lib.filter (u: elsewhere ? ${u}) readers;
          waits =
            u:
            here ? ${u}
            && lib.elem "swarm-bao-pki.service" here.${u}.after
            && lib.elem "swarm-bao-pki.service" here.${u}.wants
            && !(lib.any (n: lib.elem n here.${u}.requires) storeUnits)
            && lib.elem "container@${machine}.service" here.${u}.after
            && lib.elem "container@${machine}.service" here.${u}.wants;
          namesNoStoreUnit =
            u:
            let
              s = elsewhere.${u};
            in
            !(lib.any (n: lib.elem n (s.after ++ s.wants ++ s.requires)) storeUnits);
        in
        lib.all waits readers
        && lib.length readersHere == 10
        && lib.elem "swarm-bao-grafana-admin" readersHere
        && lib.length readersElsewhere == 7
        && lib.all namesNoStoreUnit readersElsewhere
        && !(lib.any (u: storeOnlyNoHive.systemd.services ? ${u}) readers);
    }
  ];
in
runGroup "swarm-services-switch" cases