hyperhive/nix/module-eval/core-toggle.nix

# `checks.module-eval-core-toggle` — see ./lib.nix for the shared
# rationale (why this suite exists, naming convention, "evaluates
# not executes").
{
  pkgs,
  lib,
  self,
  nixosSystem,
}:
let
  inherit
    (import ./lib.nix {
      inherit
        pkgs
        lib
        self
        nixosSystem
        ;
    })
    hive
    runGroup
    baoNames
    baoStream
    bridgePorts
    bridgePortsOrNone
    swarmServiceEnables
    ;

  allLocal = hive { deploy.singleHostSwarm = true; };

  bare = hive { };

  # The same stub with the central toggle (`deploy.hive-controller.enable`)
  # off. Paired with `bare` below to pin the defaults that do not read it:
  # each is asserted to hold the SAME literal in both, so a future edit
  # that quietly introduces the dependency — or that changes what the
  # default renders for a hive with the toggle on — fails here. Reading an
  # option off this fixture forces that option only, not the config, so the
  # toggle being off costs nothing. Also the "installs the modules and turns
  # nothing on" host the swarm-service absences below read: none of the
  # per-service deployment toggles derives from the hive being on, so it
  # renders the same absences `bare` does.
  centralToggleOff = hive { deploy.hive-controller.enable = false; };

  # Every hive defaults the agents' queue address to the queue's name, so the
  # only hive without one is a hive told to have none.
  noAgentQueue = hive { deploy.hive-controller.queue.agentNatsUrl = null; };

  # On the forge's host: the runner is refused anywhere else.
  withCi = hive {
    deploy.forgejo.enable = true;
    deploy.forgejo.ci.enable = true;
  };

  # A hive whose one gateway-published swarm service sits on another host:
  # every swarm name still configured, not one of them served here.
  forgeElsewhere = hive { deploy.forgejo.behindGateway = false; };

  # A priority collision is a property of the *option*, not
  # of the merged value's interior — nix throws the moment the value is
  # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is
  # both necessary and sufficient. `deepSeq` over-specifies this: it keeps
  # walking *into* the resulting value after the merge already succeeded,
  # and a package/derivation-shaped value's `override`/`overrideAttrs`
  # self-reference sends it into nixpkgs' fixpoint machinery and blows the
  # stack (measured — this is not a hypothetical).
  forceCiServiceConfigs =
    let
      svcs = withCi.containers.hive-ci.config.systemd.services;
      vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs);
    in
    builtins.foldl' (acc: v: builtins.seq v acc) true vals;
  cases = [
    {
      # Both halves matter. The equality is the "no longer consults the central
      # toggle" half; the literal is the "and still renders what it always
      # did" half, which an equality on its own would let drift to `false` in
      # lockstep.
      name = "the forge's behindGateway default is true regardless of the central toggle";
      ok =
        bare.services.hyperhive.deploy.forgejo.behindGateway == true
        && centralToggleOff.services.hyperhive.deploy.forgejo.behindGateway == true;
    }
    {
      # Downstream of the one above — publicUrl reads `behindGateway`, so it
      # tracked the central toggle transitively as well as directly. The domain
      # is the stub's swarm domain, which both fixtures share.
      name = "the forge's publicUrl default follows behindGateway alone, not the central toggle";
      ok =
        bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"
        && centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local";
    }
    {
      # And that it still tracks `behindGateway` at all: without this arm the
      # case above passes just as well for a default hardcoded to the URL.
      name = "the forge's publicUrl default is still null with behindGateway off";
      ok =
        (hive { deploy.forgejo.behindGateway = false; }).services.hyperhive.swarm.forge.publicUrl == null;
    }
    {
      # The controller's token path follows where the forge runs
      # (./forge-placement.nix), never the central toggle: a forge host with
      # the toggle off still renders the path.
      name = "the swarm controller's forgeTokenFile default does not consult the central toggle";
      ok =
        (hive {
          deploy.hive-controller.enable = false;
          deploy.forgejo.enable = true;
        }).services.hyperhive.deploy.swarm-controller.forgeTokenFile
        == "/var/lib/hyperhive-forge/swarm-controller.token";
    }
    {
      name = "a hive that does not host the swarm's shared services runs none of them";
      ok =
        let
          es = swarmServiceEnables bare;
        in
        lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es);
    }
    {
      name = "a hive that has not opted into all-local runs no swarm controller";
      ok = !bare.services.hyperhive.deploy.swarm-controller.enable;
    }
    {
      name = "the all-local mode turns the swarm controller on";
      ok = allLocal.services.hyperhive.deploy.swarm-controller.enable;
    }
    {
      # Where the reader puts the files and where the daemon looks for them is
      # one agreement spanning two modules. Asserted against the option rather
      # than the literal so moving the directory moves both ends.
      name = "hive-c0re is told where the agents' queue credential lands";
      ok =
        allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR
        == toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir;
    }
    {
      # Inside an agent container loopback is the agent itself, so the agents'
      # address must never be one. By name it is the same string the hive
      # dials, which ./nats-tls.nix pins for every client.
      name = "the agents' queue address is the queue's name, never loopback";
      ok =
        let
          e = allLocal.systemd.services.hive-c0re.environment;
        in
        e.HIVE_AGENT_NATS_URL == "tls://nats.t.local:4222"
        && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL)
        && e.HIVE_AGENT_NATS_URL == e.HIVE_C0RE_NATS_URL;
    }
    {
      # The agents mint against the swarm's IdP, the same endpoint the hive's
      # own client uses — a hive-local guess would produce a token the queue
      # would not accept.
      name = "the agents' token endpoint is the swarm IdP's";
      ok =
        let
          e = allLocal.systemd.services.hive-c0re.environment;
        in
        lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT
        && e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT;
    }
    {
      # The absence arm, and what makes the two above able to fail: a hive
      # with no queue address for its agents must forward neither coordinate,
      # because half a pair reaches the harness as a partial configuration
      # rather than as none.
      name = "a hive with no agent queue address forwards no agent queue coordinates";
      ok =
        let
          e = noAgentQueue.systemd.services.hive-c0re.environment;
        in
        !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT);
    }
    {
      # Where the store is and where the agent is told it is, one agreement
      # spanning two modules. Asserted against the hive's own `BAO_ADDR`
      # rather than a literal, because an agent pointed at a different
      # spelling of the same store presents a certificate to a listener whose
      # name it cannot verify.
      name = "an agent is told the same store address its hive uses";
      ok =
        let
          e = allLocal.systemd.services.hive-c0re.environment;
        in
        e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR;
    }
    {
      # A hive with no certificate of its own can collect no agent's identity,
      # so forwarding an address would name a store nothing in the container
      # can reach. The same gate the `BAO_*` pair beside it sits behind.
      name = "a hive with no store identity forwards no store address to its agents";
      ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR);
    }
    {
      # The gateway's per-name issuer choice. If this ever collapses to a
      # constant, every swarm-service vhost serves a certificate its CA
      # is name-constrained out of — which evaluates cleanly and fails in
      # a browser.
      name = "a swarm service name gets the swarm-services leaf and the default server does not";
      ok =
        let
          l = allLocal.services.hyperhive.gateway.lib;
        in
        (l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate;
    }
    {
      # A leaf is a key that can SERVE every name in it, so the names it
      # asks for are that key's blast radius. The forge's host fronts forge
      # and nothing else; the last conjunct is the control, since
      # `auth.t.local` is in the swarm's set and an unnarrowed request would
      # carry it here too.
      name = "a gateway's services leaf asks for only the swarm names that host fronts";
      ok =
        let
          forgeHere = hive { deploy.forgejo.enable = true; };
          s = forgeHere.services.hyperhive.swarm.serviceDomains;
          u = forgeHere.systemd.services.swarm-services-cert.script;
        in
        lib.hasInfix "alt_names=forge.t.local" u
        && !(lib.hasInfix "auth.t.local" u)
        && lib.elem "auth.t.local" s;
    }
    {
      # The narrowing's floor: no names left means no request, rather than a
      # request for an empty SAN set that the store would refuse. Its own
      # vhosts are the hive leaf's, which this host still signs.
      name = "a host fronting none of the swarm's service names requests no services leaf";
      ok =
        forgeElsewhere.services.hyperhive.swarm.localServiceDomains == [ ]
        && forgeElsewhere.services.hyperhive.swarm.serviceDomains != [ ]
        && lib.hasInfix "want_svc=0" forgeElsewhere.systemd.services.swarm-services-cert.script;
    }
    {
      # nixos asserts when a vhost declares both, so this is also a
      # statement that the `removeAttrs` upstream of it still happens.
      name = "the swarm UI vhost forces TLS instead of merely adding it";
      ok =
        let
          v = allLocal.services.nginx.virtualHosts."t.local";
        in
        v.forceSSL && !(v.addSSL or false);
    }
    {
      name = "a hive with matrix off serves no matrix discovery endpoint";
      ok =
        !(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations);
    }
    {
      # main got eval-borked twice by this exact class of bug (once on the
      # unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to
      # `gitea-actions-runner.nix` adds a plain `serviceConfig.*`
      # definition that collides with one of ours, and nix refuses to
      # merge two plain definitions at *host* eval. No other check
      # instantiates a host with `containers.hive-ci` actually enabled, so
      # the collision only surfaces on operator deploy, not in CI.
      name = "the CI container's unit definitions merge without a priority collision";
      ok = forceCiServiceConfigs;
    }
    {
      # The collector reaches these routes through the gateway now, so each
      # store needs an ingest location of its own. Without one the write rides
      # the `/` catch-all: unauthenticated on the metrics store, and into a
      # browser redirect on the log store.
      name = "each store's vhost has an authenticated ingest location";
      ok =
        let
          v = allLocal.services.nginx.virtualHosts;
          m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null;
          l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null;
        in
        m != null
        && l != null
        && lib.hasInfix "auth_request" m.extraConfig
        && lib.hasInfix "auth_request" l.extraConfig;
    }
    {
      # The arm that actually protects something. A pusher handed
      # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page,
      # which answers 200 — ingest reporting healthy while storing nothing.
      # The third clause is the positive control: the log store's browser
      # location really does redirect, so this says the machine routes differ
      # rather than that the string is absent from the whole file.
      name = "the ingest locations answer 401 instead of redirecting a pusher";
      ok =
        let
          v = allLocal.services.nginx.virtualHosts;
          m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig;
          l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig;
          browser = v."logs.t.local".locations."/".extraConfig;
        in
        !(lib.hasInfix "error_page" m)
        && !(lib.hasInfix "error_page" l)
        && lib.hasInfix "error_page" browser;
    }
    {
      # The read counterpart to the ingest location: an agent queries the log
      # store with a bearer token, and the `/` catch-all is the browser's
      # route. Riding it would mean inheriting the login redirect the next
      # case is about, so the route has to exist separately to be gated
      # separately.
      name = "the log store's vhost has an authenticated machine query location";
      ok =
        let
          q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null;
        in
        q != null && lib.hasInfix "auth_request" q.extraConfig;
    }
    {
      # Same trap as the ingest case, on the read side, where it is worse: a
      # redirected pusher at least stores nothing visibly, while a redirected
      # *reader* is handed a 200 carrying login HTML and records a query that
      # succeeded and matched no logs. The browser clause is the positive
      # control — that location really does redirect to the login host — so a
      # pass means these two routes differ rather than that the strings are
      # absent from the whole vhost.
      name = "the machine query location answers 401 instead of redirecting to a login page";
      ok =
        let
          v = allLocal.services.nginx.virtualHosts;
          q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig;
          browser = v."logs.t.local".locations."/".extraConfig;
        in
        !(lib.hasInfix "error_page" q)
        && !(lib.hasInfix "auth.t.local" q)
        && lib.hasInfix "error_page" browser
        && lib.hasInfix "auth.t.local" browser;
    }
    {
      # Read access is deliberately unscoped: an authenticated caller reads
      # the whole swarm's logs until a permission system exists. Pinned so a
      # scoping parameter arriving later is a visible diff here rather than a
      # quiet change of rule — and pinned on `proxyPass` too, because
      # VictoriaLogs takes its filters as request parameters, which ride an
      # upstream URI as easily as a directive. The first clause is the
      # control: it proves the location resolved and that `hasInfix` finds
      # what is genuinely in this string, so the absences below mean absent
      # rather than unreadable.
      name = "the machine query location forwards the caller's query unmodified";
      ok =
        let
          q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/";
        in
        lib.hasInfix "auth_request" q.extraConfig
        && !(lib.hasInfix "extra_filters" q.extraConfig)
        && !(lib.hasInfix "extra_stream_filters" q.extraConfig)
        && !(lib.hasInfix "$args" q.extraConfig)
        && !(lib.hasInfix "?" q.proxyPass);
    }
    {
      # The absence arm for the case above, and the option's own rule — a
      # service declares its entry under its own `enable` — made checkable.
      # Without it, moving the assignment outside the collector's `mkIf`
      # passes every arm above while handing a collector-less hive a scrape
      # target for a port nothing binds.
      name = "a hive with no collector declares no self-scrape target";
      ok = bare.services.hyperhive.otel.scrapeTargets == { };
    }
    {
      # Absence arm, and the one that matters: claiming a name this host does
      # not serve points every local reader at the wrong machine.
      name = "a hive that does not run the store claims no name for it";
      ok = !(builtins.elem "bao.t.local" (baoNames bare));
    }
    {
      # ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a
      # host-wide option, so a block rendered outside the store's own `mkIf`
      # gives every hive in the swarm a listener — on the port the store
      # answers on, in front of no store at all.
      name = "a hive that does not run the store renders no stream passthrough";
      ok = baoStream bare == "";
    }
    {
      # Absence arm for the case above — a hive with no store has no reason to
      # open the store's port, and opening it would point agents at a host that
      # answers nothing.
      name = "a hive that does not run the store opens no bridge port for it";
      ok = !(builtins.elem 8200 (bridgePorts bare));
    }
    {
      # The four blocks below used to be gated on the hive being enabled AND
      # their own condition. The second half was always the load-bearing one —
      # none of these conditions is derived from the hive toggle — so these
      # arms pin what the conjunct was doing: nothing. Each is written against
      # a host with the hive OFF as well as one with it on, because the way a
      # dropped conjunct fails is by making something unconditional, and that
      # shows up as a service appearing where nothing asked for it.
      name = "no bridge port is opened for an exposeHostPorts nobody set";
      ok =
        !(builtins.elem 5432 (bridgePortsOrNone bare))
        && !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff));
    }
    {
      # `deploy.swarm-controller.enable`, which defaults false and is
      # deliberately not derived from the hive toggle — a swarm has one
      # controller, so the host that runs it says so itself.
      #
      # Probed by what the daemon needs in order to run, not by
      # `? swarm-controller`: ./host-modules/hive-tls.nix defines an
      # environment key on that unit name, which leaves the attr
      # present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive
      # that has a CA — see the comment there. The credential oneshot has no
      # second definer, so its absence is the unambiguous half.
      name = "the swarm controller does not run unless this host is told to run it";
      ok =
        let
          inert =
            machine:
            !(machine.systemd.services ? swarm-controller-credential)
            && !(
              (machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart
            );
        in
        inert bare && inert centralToggleOff;
    }
    {
      # `deploy.swarm-otel.enable`, same shape: the swarm's collector is one
      # host's job, and the container is the whole of what it renders.
      name = "the swarm collector container is absent unless this host is told to run it";
      ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel);
    }
    {
      # `deploy.swarm-ui.enable`, which defaults to the controller's toggle —
      # derived from a sibling deployment decision, still not from the hive
      # toggle. `t.local` is the fixtures' swarm domain, which is the apex the
      # UI claims; the arm below is what proves this vhost renders at all.
      name = "the swarm UI vhost is absent unless this host is told to serve it";
      ok =
        !(bare.services.nginx.virtualHosts ? "t.local")
        && !(centralToggleOff.services.nginx.virtualHosts ? "t.local");
    }
    {
      # The three infrastructure toggles are off by default and asserted by
      # whoever needs them. With nothing on the host needing them, none of
      # the three renders — which is also the control for the arm below.
      name = "the gateway, resolver and bridge are absent where nothing on the host needs them";
      ok =
        !centralToggleOff.services.hyperhive.gateway.enable
        && !centralToggleOff.services.hyperhive.gateway.dns.enable
        && !centralToggleOff.services.hyperhive.network.enable
        && !(centralToggleOff.services.nginx.enable or false)
        && !(centralToggleOff.services.dnsmasq.enable or false)
        && !(centralToggleOff.networking.bridges ? hive-br0);
    }
    {
      # hive-c0re asserts all three, and it follows the central toggle — so
      # an ordinary hive keeps getting them with no opt-in, which is what
      # this change must not break.
      name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them";
      ok =
        bare.services.hyperhive.gateway.enable
        && bare.services.hyperhive.gateway.dns.enable
        && bare.services.hyperhive.network.enable
        && bare.services.nginx.enable
        && bare.services.dnsmasq.enable
        && bare.networking.bridges ? hive-br0;
    }
    {
      # A host config written against either old spelling still runs its hive.
      # The builder's own `true` is lowered to `mkDefault false`, so only the
      # forwarded definition can turn it on.
      name = "the old hive toggles forward to deploy.hive-controller.enable and warn";
      ok =
        let
          renamedFrom =
            old: m:
            lib.any (
              w: lib.hasInfix old w && lib.hasInfix "services.hyperhive.deploy.hive-controller.enable" w
            ) m.warnings;
          viaOld =
            extra:
            hive (
              extra
              // {
                deploy.hive-controller.enable = lib.mkDefault false;
              }
            );
          oldEnable = viaOld { enable = true; };
          oldC0re = viaOld { c0re.enable = true; };
        in
        oldEnable.services.hyperhive.deploy.hive-controller.enable
        && oldEnable.systemd.services ? hive-c0re
        && renamedFrom "services.hyperhive.enable" oldEnable
        && oldC0re.services.hyperhive.deploy.hive-controller.enable
        && oldC0re.systemd.services ? hive-c0re
        && renamedFrom "services.hyperhive.c0re.enable" oldC0re;
    }
    {
      # Refused wherever something turns the name into an identifier, and only
      # there. The store host is the control that the refusal is not the hive's
      # alone; the host running neither is the control that it is gated at all.
      name = "a missing hiveName is refused on a hive and on a store host, not elsewhere";
      ok =
        let
          refusedHiveName =
            extra:
            lib.any (a: !a.assertion && lib.hasInfix "services.hyperhive.hiveName to be set" a.message)
              (hive ({ hiveName = null; } // extra)).assertions;
        in
        refusedHiveName { }
        && refusedHiveName {
          deploy.hive-controller.enable = false;
          deploy.bao.enable = true;
        }
        && !(refusedHiveName { deploy.hive-controller.enable = false; });
    }
  ];
in
runGroup "core-toggle" cases