From dc418a52236a4357d2b7a3e3034de0a7c20081f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?m=C3=BCde?= Date: Sun, 20 Sep 2026 04:25:54 +0200 Subject: [PATCH] nix: split module-eval into per-subsystem checks The single module-eval derivation forced ~62 full nixosSystem fixtures live at once to compute its cases list: 10.6GB peak RSS / 5m25s to evaluate, by far the dominant cost in nix flake check. Splits it into 21 independent checks.module-eval-* derivations (1-7 fixtures each) sharing builders/helpers via module-eval/lib.nix, so no single derivation needs more than a handful of fixtures live at once. A few cases spanning two clusters carry a small duplicated fixture rather than threading shared state through lib.nix. --- nix/checks.nix | 96 +- nix/module-eval.nix | 2992 --------------------- nix/module-eval/agent-icon.nix | 49 + nix/module-eval/agent-matrix.nix | 107 + nix/module-eval/agent-memory.nix | 87 + nix/module-eval/agent-otel.nix | 95 + nix/module-eval/agent-plugins.nix | 89 + nix/module-eval/agent-queue-bao.nix | 169 ++ nix/module-eval/bao-basics.nix | 219 ++ nix/module-eval/bao-controller.nix | 216 ++ nix/module-eval/bao-grants.nix | 236 ++ nix/module-eval/bao-matrix-reader.nix | 224 ++ nix/module-eval/bao-otel-collector.nix | 131 + nix/module-eval/core-toggle.nix | 422 +++ nix/module-eval/grafana.nix | 222 ++ nix/module-eval/hive-otel.nix | 129 + nix/module-eval/lib.nix | 202 ++ nix/module-eval/matrix-core.nix | 79 + nix/module-eval/name-guards.nix | 115 + nix/module-eval/nats-authelia.nix | 121 + nix/module-eval/secret-publisher.nix | 302 +++ nix/module-eval/swarm-otel-core.nix | 160 ++ nix/module-eval/swarm-otel-identity.nix | 167 ++ nix/module-eval/swarm-services-switch.nix | 128 + 24 files changed, 3760 insertions(+), 2997 deletions(-) delete mode 100644 nix/module-eval.nix create mode 100644 nix/module-eval/agent-icon.nix create mode 100644 nix/module-eval/agent-matrix.nix create mode 100644 nix/module-eval/agent-memory.nix create mode 100644 nix/module-eval/agent-otel.nix create mode 100644 nix/module-eval/agent-plugins.nix create mode 100644 nix/module-eval/agent-queue-bao.nix create mode 100644 nix/module-eval/bao-basics.nix create mode 100644 nix/module-eval/bao-controller.nix create mode 100644 nix/module-eval/bao-grants.nix create mode 100644 nix/module-eval/bao-matrix-reader.nix create mode 100644 nix/module-eval/bao-otel-collector.nix create mode 100644 nix/module-eval/core-toggle.nix create mode 100644 nix/module-eval/grafana.nix create mode 100644 nix/module-eval/hive-otel.nix create mode 100644 nix/module-eval/lib.nix create mode 100644 nix/module-eval/matrix-core.nix create mode 100644 nix/module-eval/name-guards.nix create mode 100644 nix/module-eval/nats-authelia.nix create mode 100644 nix/module-eval/secret-publisher.nix create mode 100644 nix/module-eval/swarm-otel-core.nix create mode 100644 nix/module-eval/swarm-otel-identity.nix create mode 100644 nix/module-eval/swarm-services-switch.nix diff --git a/nix/checks.nix b/nix/checks.nix index b99155c6..b8513be0 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -16,12 +16,98 @@ in { formatting = treefmt-eval.config.build.check self; - # The only check here that covers **nix**. Every other one is a Rust + # The only checks here that cover **nix**. Every other one is a Rust # derivation, so a `.nix`-only diff moves no hash and the whole set is - # cache hits — green without evaluating what changed. See the file's - # header for what belongs in it and what needs something that executes - # rather than evaluates. - module-eval = import ./module-eval.nix { + # cache hits — green without evaluating what changed. See + # ./module-eval/lib.nix for what belongs in this suite and what needs + # something that executes rather than evaluates. + # + # Split into one derivation per subsystem cluster (./module-eval/*.nix) + # rather than one `module-eval` derivation: the combined version forced + # ~62 full `nixosSystem` fixtures live at once to compute its single + # `cases` list, measured at 10.6GB peak RSS / 5m25s to evaluate. Each + # cluster below only holds its own handful of fixtures. + module-eval-core-toggle = import ./module-eval/core-toggle.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-swarm-services-switch = import ./module-eval/swarm-services-switch.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-nats-authelia = import ./module-eval/nats-authelia.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-matrix-core = import ./module-eval/matrix-core.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-grafana = import ./module-eval/grafana.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-bao-basics = import ./module-eval/bao-basics.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-bao-matrix-reader = import ./module-eval/bao-matrix-reader.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-bao-grants = import ./module-eval/bao-grants.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-bao-controller = import ./module-eval/bao-controller.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-secret-publisher = import ./module-eval/secret-publisher.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-bao-otel-collector = import ./module-eval/bao-otel-collector.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-swarm-otel-core = import ./module-eval/swarm-otel-core.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-swarm-otel-identity = import ./module-eval/swarm-otel-identity.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-hive-otel = import ./module-eval/hive-otel.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-name-guards = import ./module-eval/name-guards.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-otel = import ./module-eval/agent-otel.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-queue-bao = import ./module-eval/agent-queue-bao.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-memory = import ./module-eval/agent-memory.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-plugins = import ./module-eval/agent-plugins.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-icon = import ./module-eval/agent-icon.nix { + inherit pkgs self nixosSystem; + inherit (pkgs) lib; + }; + module-eval-agent-matrix = import ./module-eval/agent-matrix.nix { inherit pkgs self nixosSystem; inherit (pkgs) lib; }; diff --git a/nix/module-eval.nix b/nix/module-eval.nix deleted file mode 100644 index 263ee2c3..00000000 --- a/nix/module-eval.nix +++ /dev/null @@ -1,2992 +0,0 @@ -# `checks.module-eval` — the flake check that covers **nix**. -# -# Every other check in ./checks.nix is a Rust derivation, so a `.nix`-only -# diff moves no hash and `nix flake check` goes green **without evaluating -# what changed**. This derivation's hash is a function of the results -# below, so a change that flips a property rebuilds it and fails naming it. -# -# Anything expressible as a module `assertion` **should be one instead** — -# an assertion fires at deploy time for a real operator, not only in CI. -# What cannot is the **absence class**: "a hive that hasn't opted in -# renders what it did before", "this unit does not exist unless X" — claims -# about the rendered config rather than about a config being invalid. -# -# ⚠️ **Name a case for the PROPERTY it defends, never the ticket that -# prompted it**; a case named after a ticket has the ticket's lifetime. -# -# ⚠️ **This evaluates, it does not execute** — a command line, a request or -# a certificate needs something that *runs* it. A case needing a rendered -# file must stub what it drags in (`deploy.swarm-ui.package = pkgs.emptyDirectory`). -{ - pkgs, - lib, - self, - nixosSystem, -}: -let - # Stub host, same shape ./docs/default.nix already uses: enough for a - # `nixosSystem` to evaluate, nothing that pulls a real disk or - # bootloader in. - hive = - extra: - (nixosSystem { - system = pkgs.stdenv.hostPlatform.system; - modules = [ - self.nixosModules.default - { - fileSystems."/" = { - device = "/dev/null"; - fsType = "tmpfs"; - }; - boot.loader.grub.enable = false; - system.stateVersion = "25.11"; - # `recursiveUpdate`, not `//`: a plain `//` only merges the - # *top-level* keys of `extra` in, so an `extra` that touches a - # nested attr under an existing top-level key (e.g. `swarm.*`) - # silently drops every sibling under that key instead of merging - # into it — the same class of bug as the `services.hyperhive` - # double-nesting mistake this stub already had to dodge once, - # just one level down. `recursiveUpdate` merges nested attrsets - # all the way down instead. - services.hyperhive = lib.recursiveUpdate { - enable = true; - hiveName = "h1"; - swarm.domain = "t.local"; - swarm.hives.h1.domain = "h1.t.local"; - } extra; - } - ]; - }).config; - - # The other half of the tree. `nix/agent-modules/` is evaluated by nothing - # else in this suite — every fixture above is a host — so a rendered - # container config was only ever read by a real deploy. Same entry point - # the meta flake hands a container, so what this evaluates is what an - # agent gets. - # - # Takes a whole module rather than a settings attrset, so a fixture can - # reach either spelling of the agent tier. `user.name` is the one option - # with no usable default, set here at its real path and at `mkDefault` so a - # fixture naming its own agent still wins. - agentWith = - module: - (nixosSystem { - system = pkgs.stdenv.hostPlatform.system; - modules = [ - self.nixosModules.agent-base - { - fileSystems."/" = { - device = "/dev/null"; - fsType = "tmpfs"; - }; - boot.loader.grub.enable = false; - system.stateVersion = "25.11"; - services.hyperhive.agent.user.name = lib.mkDefault "a1"; - } - module - ]; - }).config; - - # Note `hyperhive`, not `services.hyperhive`: the agent tier's options moved - # under `services.hyperhive.agent`, and ../agent-modules/renamed-options.nix - # keeps the top-level spelling reaching them. ⚠️ That shim covers the - # options that existed when the tier moved and nothing since, so a fixture - # for an option added afterwards has to go through [`agentWith`] and name - # the real path. - agent = extra: agentWith { hyperhive = extra; }; - - allLocal = hive { deploy.singleHostSwarm = true; }; - bare = hive { }; - - # The same stub with the central toggle off. Paired with `bare` below to pin - # the defaults that used to read `services.hyperhive.enable` and no longer - # do: each is asserted to hold the SAME literal in both, so a future edit - # that quietly re-introduces the dependency — or that changes what the - # default renders for a hive with the toggle on — fails here. Reading an - # option off this fixture forces that option only, not the config, so the - # toggle being off costs nothing. Also the "installs the modules and turns - # nothing on" host the swarm-service absences below read: none of the - # per-service deployment toggles derives from the hive being on, so it - # renders the same absences `bare` does. - centralToggleOff = hive { enable = false; }; - - # The swarm-services toggle with the central one off, so the only thing - # that can enable the gateway/resolver/bridge here is that toggle's own - # module — every other module that asserts them is behind `enable`. - swarmServicesOnly = hive { - enable = false; - deploy.allSwarmServices = true; - }; - - withCi = hive { deploy.forgejo.ci.enable = true; }; - - # The queue's callout identity, fourth split slice. `autoGenerateCallout` is - # left FALSE on purpose: that is what makes the seed paths the thing deciding - # `responderConfigured`, so the assertion below is about the seeds rather - # than about the auto-mint branch. Every one of the seven old paths is - # defined — `enable` included, which is why it is spelled the old way here - # while the fixture below uses the new one — so dropping any single nats - # shim fails the eval, not just the arms read. - natsOldPath = hive { - swarm.nats.enable = true; - swarm.nats.autoGenerateCallout = false; - swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; - swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; - swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; - swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; - swarm.nats.authPackage = pkgs.emptyDirectory; - }; - - # Seventh split slice, plus slice 10's two authelia packages. Of slice 7's - # movers only `usersFile` has a rename entry — the other two are `readOnly`, - # and a rename module contributes a definition, which a read-only option - # refuses; see ./host-modules/deploy.nix. `package` and `bridgePackage` are - # ordinary options, so they do carry one. The two arms - # below have different jobs. `usersFile` tests the rename; the nats one tests - # that a reader repointed to the new namespace still renders the derived - # path, which is the failure this slice could actually have shipped — seven - # of those reads went through an alias a path-shaped grep cannot see. - autheliaOldPath = hive { - deploy.authelia.enable = true; - deploy.nats.enable = true; - swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml"; - swarm.authelia.package = pkgs.emptyDirectory; - swarm.authelia.bridgePackage = pkgs.emptyDirectory; - swarm.nats.autoGenerateCallout = false; - swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; - swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; - swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; - swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; - }; - - # A deliberately odd `maxRequestSize`, so the number the assertion looks for - # cannot have come from a default or from another module's literal. - matrixBodyCap = hive { - deploy.singleHostSwarm = true; - deploy.matrix.maxRequestSize = 99000000; - }; - - grafanaOldPath = hive { - deploy.grafana.enable = true; - swarm.grafana.socketDir = "/run/test-grafana-sock"; - swarm.grafana.datasourceUrl = "http://127.0.0.1:19999"; - swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998"; - swarm.grafana.plugins = [ ]; - swarm.grafana.package = pkgs.emptyDirectory; - }; - - # The metrics UI beside the IdP. It reads its secret out of the store like - # every other Grafana host, so it needs a store identity like every other - # Grafana host — the cert pair here is not scenery, it is the arm that would - # have caught the deleted co-located copy unit coming back. - grafanaWithAuthelia = hive { - deploy.grafana.enable = true; - deploy.grafana.plugins = [ ]; - deploy.grafana.package = pkgs.emptyDirectory; - deploy.authelia.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - # The same UI with the IdP on ANOTHER host and a store leaf placed by hand. - # Knowing an IdP is not running one: `swarm.authelia.url` is what says this - # swarm has SSO, and nothing about this host does. Identical to the fixture - # above in everything the delivery path reads, which is the point. - grafanaRemoteAuthelia = hive { - deploy.grafana.enable = true; - deploy.grafana.plugins = [ ]; - deploy.grafana.package = pkgs.emptyDirectory; - swarm.authelia.url = "https://auth.example.invalid"; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - # A Grafana host holding no store identity. This used to be the shape the - # module went QUIET on — no OIDC block, a warning, and a container whose - # login form is off regardless, so no way in and nothing failed. It is kept - # rather than deleted because the shape is still reachable by an operator; - # what changed is the deliverable, from a warning nothing reads back to a - # refusal naming the two options to set. Only the identity is missing, so an - # arm below can name which refusal fired. - grafanaNoIdentity = hive { - deploy.grafana.enable = true; - deploy.grafana.plugins = [ ]; - deploy.grafana.package = pkgs.emptyDirectory; - swarm.authelia.url = "https://auth.example.invalid"; - }; - # The mirror image: the identity is placed, and the swarm names no IdP. The - # other half of "SSO must always be configured", and isolated the same way — - # exactly one thing wrong, so the arm reads one refusal. - grafanaNoSso = hive { - deploy.grafana.enable = true; - deploy.grafana.plugins = [ ]; - deploy.grafana.package = pkgs.emptyDirectory; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - - # Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its - # two reasons. An assertion is a config VALUE until something forces it — - # `.config` never throws — so a fixture in a state the module refuses is - # evaluable and the refusal is readable as data. That is what lets a case - # check that a misconfiguration is REPORTED, rather than only that it is not - # silently accepted. - # - # Matched on the option name the message names, not on its prose, so the - # wording stays rewordable: the option name is the part an operator has to - # act on, and a message that stopped naming it would be the actual defect. - grafanaRefusedFor = - m: option: - lib.any ( - a: - !a.assertion - && lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message - && lib.hasInfix option a.message - ) m.assertions; - - baoPkcs11 = hive { - deploy.bao.enable = true; - deploy.bao.seal = "pkcs11"; - }; - baoShamir = hive { - deploy.bao.enable = true; - deploy.bao.seal = "shamir"; - }; - baoExplicitCerts = hive { - deploy.bao.enable = true; - deploy.bao.serverCertFile = "/etc/pki/bao.pem"; - deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem"; - }; - # The store and a service that reads from it, versus the store alone. The - # pair is what makes the reader's absence arm mean anything. - baoWithMatrix = hive { - deploy.bao.enable = true; - deploy.matrix.enable = true; - }; - # A hive that reads from a store it does not run: no `deploy.bao.enable`, so - # nothing here mints a leaf and the operator names one placed by hand. The - # deployment this pairing exists to serve, and the one that was previously - # inexpressible — the gate asked whether the store was a neighbour. - baoRemoteReader = hive { - deploy.matrix.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - # The same hive with the identity taken away, which separates "a homeserver - # is deployed" from "this host can authenticate to the store". - matrixNoBaoIdentity = hive { deploy.matrix.enable = true; }; - - # The store, plus a placed bootstrap token: the only shape in which the - # swarm's first grant can be written at all. - baoGrantHere = hive { - deploy.bao.enable = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; - }; - # The credential without the store. Writing the first grant is a store-side - # operation, so a host holding only the token has nothing to do — and this - # is the arm that separates "an operator placed a token" from "this box can - # act on it". - baoGrantNoStore = hive { - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; - }; - # Store and controller on one machine, with a CN no default could supply. - # The odd value is what lets the case below tell "both ends read the same - # option" from "both ends happen to say swarm-controller". - baoControllerHere = hive { - deploy.bao.enable = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; - deploy.bao.controllerCommonName = "cn-marker-not-a-default"; - deploy.swarm-controller.enable = true; - }; - # The controller with no store, which is every spread deployment. Nothing - # mints here, so the pairing must leave the paths unset rather than name - # files this host will never have. - controllerNoStore = hive { deploy.swarm-controller.enable = true; }; - - # The two authorities told apart. A deployment that self-signs both ends - # points `clientCaFile` and `serverCaFile` at one file, so on the fixture - # above the CA a hive is issued from and the CA the store is verified by are - # the same string — and a case wiring either into the other's slot passes. - # This is the deployment where they differ, which is what makes the arm - # below able to fail at all. - controllerTwoCas = hive { - deploy.bao.enable = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; - deploy.swarm-controller.enable = true; - deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem"; - deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem"; - }; - - # The IdP and the store on one machine: the shape where minted plaintext and - # a store identity are both present without an operator placing anything. - # Two hives in the roster, because the publisher walks it — an arm written - # against a single-hive fixture passes on a hardcoded name. - secretPublisherHere = hive { - deploy.bao.enable = true; - deploy.authelia.enable = true; - swarm.hives.h2.domain = "h2.t.local"; - }; - # The IdP with no store on the box and a leaf placed by hand, which is the - # deployment this unit exists for: authelia is the one host the store is - # guaranteed not to share once either has a machine of its own. - # - # ⚠️ `enable` is deliberately NOT set here. It used to be, with a comment - # saying the default asked whether both ran on this host — which documented - # the co-location bug instead of catching it. Leaving it unset is what makes - # this fixture exercise the default rather than mask it. - secretPublisherRemote = hive { - deploy.authelia.enable = true; - deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem"; - deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem"; - }; - # The same IdP with the identity taken away. Minting the secrets is not being - # able to publish them, and this is the arm that separates the two. - secretPublisherNoIdentity = hive { deploy.authelia.enable = true; }; - - # The host's `bao` wrapper, pulled apart once so each case below names one - # property instead of a conjunction — a failing conjunction says only that - # something is wrong. - baoHostPackages = controllerTwoCas.environment.systemPackages; - baoWrapper = lib.findFirst (p: (p.name or "") == "bao-hive") null baoHostPackages; - baoWrapperCmd = if baoWrapper == null then "" else (baoWrapper.buildCommand or ""); - - # The store and the token, with no CA to trust. `mkForce` because the PKI - # glue supplies one by default here — this is the deployment that brings its - # own certificates and has not named the authority yet, and it separates - # "the grant unit runs" from "cert auth can be set up". - baoGrantNoClientCa = hive { - deploy.bao.enable = true; - deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; - deploy.bao.clientCaFile = lib.mkForce null; - }; - - baoNames = machine: machine.services.hyperhive.gateway.localNames; - - # What nginx is handed for its `stream {}` block. Deliberately not - # `virtualHosts`: a vhost is the terminating shape ./host-modules/ - # swarm-bao.nix's header refuses, and this is the passthrough that is not. - baoStream = machine: machine.services.nginx.streamConfig; - - # The bridge-interface firewall — the list `network.exposeHostPorts` merges - # into, and the only place a port is opened for agents. The host's own - # `allowedTCPPorts` is a different list and a different exposure. - bridgePorts = - machine: - machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName}.allowedTCPPorts; - - # Same list, on a host that may not declare the interface at all: the - # `[ 80 443 ]` block is what creates the attr, and it is off where the hive - # is. Reading it through `bridgePorts` would throw rather than report an - # empty exposure, which is precisely the state the cases below assert. - bridgePortsOrNone = - machine: - (machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName} or { - allowedTCPPorts = [ ]; - } - ).allowedTCPPorts; - - # Ports asked for on such a host. The request is the whole condition: the - # firewall hole exists because an operator named a port, not because the - # hive is running, and an agent reaching a host service is a claim about - # the host's own listeners either way. - exposedPortsNoHive = hive { - enable = false; - network.exposeHostPorts = [ 5432 ]; - }; - - # The swarm UI where its own toggle is on — the control the absence arm - # needs, since nothing else in this suite renders this vhost and an arm - # saying "it is not there" would hold just as well if it were never there. - # Package stubbed per this file's header: the vhost roots at it. - swarmUiHere = hive { - deploy.swarm-ui.enable = true; - deploy.swarm-ui.package = pkgs.emptyDirectory; - }; - - baoTwoAddresses = hive { - deploy.bao.enable = true; - deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; - # Pinned, not incidental: the case counting these listeners is about the - # declared addresses, and a collector on this host would add one of its own. - deploy.swarm-otel.enable = false; - }; - # The store with and without a collector on the same host. `scrapeTargets` - # is only ever read by a local collector, so the metrics endpoint is a - # function of the pairing rather than of the store. - baoWithCollector = hive { - deploy.bao.enable = true; - deploy.swarm-otel.enable = true; - # A second job declared as bare `host:port`, so the pair of cases below - # reads one rendered scrape list: the store's entry carries a path, this - # one carries none. - swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; - }; - baoNoCollector = hive { - deploy.bao.enable = true; - deploy.swarm-otel.enable = false; - }; - - # The config file openbao parses, not the nix that produces it: a setting it - # requires is absent here without anything in the module system minding, so - # the daemon's own startup is otherwise the first reader. - baoSettings = machine: machine.containers.swarm-bao.config.services.openbao.settings; - - # The store's units live inside its container, so the gates below have to - # look there rather than at the host's service set. - baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; - - # The scrape list prometheus is handed, not the option a service declared: - # the address, the path and the query are one string on the way in and three - # fields on the way out, and only the second shape is what gets requested. - scrapeJob = - machine: job: - lib.findFirst (c: c.job_name == job) null - machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; - - # A swarm collector on a host that runs NEITHER store — the fully-spread - # shape from docs/swarm/services.md, and the one the old per-host gates made - # inexpressible. It is the whole point of the cases below that this hive is - # not a degenerate configuration but a supported one. - otelNoStores = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - deploy.victoriametrics.enable = false; - deploy.victorialogs.enable = false; - }; - otelSettings = - machine: machine.containers.swarm-otel.config.services.opentelemetry-collector.settings; - - # The collector beside authelia, reading its own OIDC secret out of the - # store like every other collector — the cert pair here is not scenery, it - # is the arm that would catch the deleted co-located copy unit coming back. - otelBaoWithAuthelia = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - # The same collector with the IdP on ANOTHER host and a store leaf placed by - # hand. Identical to the fixture above in everything the delivery path - # reads, which is the point. - otelBaoRemoteAuthelia = hive { - deploy.swarm-otel.enable = true; - swarm.authelia.url = "https://auth.example.invalid"; - deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; - deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; - }; - # A collector holding no store identity at all. Unlike Grafana's mirror - # image, this is not a refused shape: the collector still receives every - # hive's telemetry with nothing to push authenticated with, which is the - # already-supported degrade `haveCollectorSecret` names above the module's - # `let`. What this fixture is for is checking the reading unit itself does - # not render, rather than rendering with an env var nothing filled in. - otelNoIdentity = hive { - deploy.swarm-otel.enable = true; - swarm.authelia.url = "https://auth.example.invalid"; - deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret"; - }; - - # authelia somewhere else, the credential delivered by hand. Whether this - # collector authenticates must follow the credential, never another - # service's placement. - # - # The `swarm.otel.clientSecretFile` below is the PRE-RENAME path. It predates - # the split and is deliberately left spelled that way: it makes this fixture - # the old-path case for that option too, so dropping its rename entry fails - # the eval here rather than only in a real operator's config. - otelRemoteAuthelia = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = false; - # Where that elsewhere IS. Running no IdP does not mean knowing no IdP: - # the authenticator this fixture exists to render puts this address in its - # `token_url`, so a hive with a secret and no URL has a credential it can - # present nowhere. - swarm.authelia.url = "https://auth.example.invalid"; - swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; - }; - - # A collector whose ONLY scrape work is published: loopback targets forced - # empty, one published job declared. Unreachable in a real deploy today — - # the module seeds `scrapeTargets.collector` under its own `enable`, so the - # loopback set is never empty on its own — which is exactly why the arm - # below needs a fixture that takes that seeding away. `mkForce` is what - # does it, and it leaves the collector itself enabled: the state under test - # is a running collector with no self-scrape, not an absent one. - otelOnlyPublished = hive { - deploy.swarm-otel.enable = true; - swarm.otel.scrapeTargets = lib.mkForce { }; - swarm.otel.publishedScrapeTargets.remote = "https://remote.t.local/metrics"; - }; - - # The log path's three hops, one fixture each. Nothing carries a journal - # record end to end at eval time, so what these defend is the part no tier - # can check for itself: each hop's output is the next hop's input, and - # every mismatch between them is silent — a push accepted and routed - # nowhere, a receiver pointed at an empty directory, a pipeline that does - # not exist. - agentBridge = "http://10.42.0.1:4318"; - agentOtel = agent { - otel.enable = true; - otel.endpoint = agentBridge; - }; - # The same agent over the other wire protocol. An exporter's NAME is what - # selects it, so this is where a defined exporter and the pipeline's - # reference to it can drift apart. - agentOtelGrpc = agent { - otel.enable = true; - otel.endpoint = agentBridge; - otel.protocol = "grpc"; - }; - agentNoOtel = agent { }; - - # The agent side of the swarm queue. Both coordinates set is the only state - # in which the harness unit declares a credential at all, so the pair and - # the empty fixture beside it are the two arms worth having. - agentQueue = agent { - queue.natsUrl = "nats://10.42.0.1:4222"; - queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; - }; - agentNoQueue = agent { }; - - # The agent side of the swarm secret store. The address is the whole switch — - # it is both what generates the login check and what that check points at — - # so it and the empty fixture beside it are the two arms worth having. - agentBao = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; }; - agentNoBao = agentWith { }; - - # The memory-pressure pair. `claudeMemoryMaxBytes` is the container's own - # cap, rendered per agent by meta.rs — the capped arm is the one every - # real deploy gets, the uncapped arm is a hive that set `infinity` or a - # RAM percentage and so hands the module no byte count to size against. - agentCapped = agent { claudeMemoryMaxBytes = 8589934592; }; - agentUncapped = agent { }; - - # Matrix's enable signal, which is the account set itself — there is no - # `matrix.enable` option left to read. Three arms, because the property has - # three distinct shapes and only one of them is the common case: - # - # - a homeserver URL, which is what the module turns into a `main` account; - # - neither URL nor operator account, the state that replaced - # `matrix.enable = false`. ⚠️ **This is the arm that matters.** `main` is - # declared by the module itself, so "any account declared" would be - # trivially true — and matrix would render for every agent in every hive — - # the moment that declaration stops being gated on the URL. Nothing else in - # this suite would notice; - # - an operator account carrying its own homeserver and no hive one, which is - # matrix on with no `main` at all. - agentMatrix = agent { matrix.url = "https://chat.t.local"; }; - agentNoMatrix = agent { }; - agentMatrixExternalOnly = agent { - matrixAccounts.ccc = { - tokenFile = "/agents/a1/state/matrix-token-ccc"; - sessionDir = "/agents/a1/state/matrix-sdk-state-ccc"; - homeserver = "https://matrix.example.invalid"; - }; - }; - # `claudePlugins`'s additive-merge shape: a plain per-agent - # definition must ADD to the module's own base-set definition rather than - # replacing it, while `lib.mkForce` must still replace the whole list - # outright — the two arms below plus the unset default (read directly off - # `bare`-shaped `agent { }`, no fixture of its own needed) are the three - # cases that shape has to hold. - agentPluginsDefault = agent { }; - agentPluginsAdded = agent { claudePlugins = [ "foo@bar" ]; }; - agentPluginsForced = agent { claudePlugins = lib.mkForce [ "foo@bar" ]; }; - # The de-dup arm: an agent that names a base-set entry explicitly must not - # get it installed twice. - agentPluginsDuplicate = agent { claudePlugins = [ "base@hyperhive" ]; }; - agentPlugins = machine: machine.services.hyperhive.agent.claudePlugins; - - # An agent with an icon and no forge. Both halves of the avatar sync — the - # `.path` watcher and the oneshot it triggers — hang off the icon, but only - # the service can do anything with a forge URL, so the icon alone is the one - # input that can render half the feature. `pkgs.emptyFile` rather than a real - # SVG: the icon is only ever a gate here, and nothing this case reads - # rasterizes it. - agentIconNoForge = agentWith { services.hyperhive.agent.icon = pkgs.emptyFile; }; - - agentHarness = machine: machine.systemd.services.hive-agent; - agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon; - agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity; - agentSettings = machine: machine.services.opentelemetry-collector.settings; - - # This hive's own collector, which is a HOST service — unlike the swarm - # tier's, which lives in a container and is read through `otelSettings`. - hiveOtel = hive { - otel.enable = true; - otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; - }; - hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; - hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings; - - # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for - # the swarm tier — the option is one string, what prometheus is handed is a - # job — but this collector is a host service, so the path to it differs. - hiveScrapeJob = - job: - lib.findFirst ( - c: c.job_name == job - ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs; - - # Two hives in the roster, which no other fixture here has: every one of - # them declares `swarm.hives.h1` alone, so a per-hive arm written against - # one of those passes on a hardcoded literal. - otelTwoHives = hive { - deploy.swarm-otel.enable = true; - deploy.authelia.enable = true; - swarm.hives.h2.domain = "h2.t.local"; - }; - - # A priority collision is a property of the *option*, not - # of the merged value's interior — nix throws the moment the value is - # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is - # both necessary and sufficient. `deepSeq` over-specifies this: it keeps - # walking *into* the resulting value after the merge already succeeded, - # and a package/derivation-shaped value's `override`/`overrideAttrs` - # self-reference sends it into nixpkgs' fixpoint machinery and blows the - # stack (measured — this is not a hypothetical). - forceCiServiceConfigs = - let - svcs = withCi.containers.hive-ci.config.systemd.services; - vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs); - in - builtins.foldl' (acc: v: builtins.seq v acc) true vals; - - # The swarm's shared services hosted HERE without the all-local mode — a - # services box with hives elsewhere, the shape ./host-modules/ - # swarm-required-services.nix documents the switch for. - swarmServicesHere = hive { deploy.allSwarmServices = true; }; - - # Same, with one of those services placed on another host. Every derivation - # in that module is `mkDefault` so this stays expressible. - swarmServicesBaoElsewhere = hive { - deploy.allSwarmServices = true; - deploy.bao.enable = false; - }; - - # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own - # option and is NOT under `deploy` — spelled at the wrong path it would be - # undeclared rather than false, and a roster that quietly loses a member is - # what the count guard in the cases below exists to catch. Its membership - # here is deliberate and was the fix for a gap, not an oversight: the hive - # tier lands wherever the swarm services do. - swarmServiceEnables = - machine: - let - h = machine.services.hyperhive; - in - { - matrix = h.deploy.matrix.enable; - otel = h.otel.enable; - authelia = h.deploy.authelia.enable; - nats = h.deploy.nats.enable; - swarm-otel = h.deploy.swarm-otel.enable; - victoriametrics = h.deploy.victoriametrics.enable; - grafana = h.deploy.grafana.enable; - victorialogs = h.deploy.victorialogs.enable; - bao = h.deploy.bao.enable; - }; - - # The hive-name guards, with the collector explicitly OFF. That is the whole - # property: the guards live where `swarm.hives` is declared, so they run in a - # deployment that has a secret store and no collector — which used to skip - # them entirely, because they were assertions inside swarm-otel's own `mkIf`. - # - # ⚠️ `controllerCommonName` is overridden to a name containing NO reserved - # fragment. Its default (`swarm-controller`) contains `swarm` and is caught - # by the substring guard whatever the cert-auth arm does — so a fixture using - # the default could not tell the two apart, and the arm under test would pass - # on the neighbour's work. - hiveNamedAfterCertSubject = hive { - deploy.swarm-otel.enable = false; - deploy.bao.controllerCommonName = "ctl"; - swarm.hives.ctl.domain = "ctl.t.local"; - }; - - # The control for both arms below: same shape, a roster nothing objects to. - hiveNamesAllLegal = hive { - deploy.swarm-otel.enable = false; - deploy.bao.controllerCommonName = "ctl"; - }; - - # The reserved subjects are a LIST, and a list with one consulted element and - # one dead one looks identical from the first element's case. This fixture - # collides with the SECOND, leaving the controller's at its default. - hiveNamedAfterPublisherSubject = hive { - deploy.swarm-otel.enable = false; - deploy.bao.secretPublisherCommonName = "pubctl"; - swarm.hives.pubctl.domain = "p.t.local"; - }; - - hiveNameWithComposedWord = hive { - deploy.swarm-otel.enable = false; - swarm.hives."h1-agent".domain = "a.t.local"; - }; - - # Markers from `lib/name-guards.nix`'s two `problem` strings. Matching the - # problem rather than the `why` prose keeps the messages rewordable. - equalityGuardFired = - h: lib.any (a: !a.assertion && lib.hasInfix "has reserved name(s)" a.message) h.assertions; - fragmentGuardFired = - h: - lib.any ( - a: !a.assertion && lib.hasInfix "has name(s) containing a reserved word" a.message - ) h.assertions; - - # Each case: a name stating the property, and `ok`. - cases = [ - { - # Both halves matter. The equality is the "no longer consults the central - # toggle" half; the literal is the "and still renders what it always - # did" half, which an equality on its own would let drift to `false` in - # lockstep. - name = "the forge's behindGateway default is true regardless of the central toggle"; - ok = - bare.services.hyperhive.deploy.forgejo.behindGateway == true - && centralToggleOff.services.hyperhive.deploy.forgejo.behindGateway == true; - } - { - # Downstream of the one above — publicUrl reads `behindGateway`, so it - # tracked the central toggle transitively as well as directly. The domain - # is the stub's swarm domain, which both fixtures share. - name = "the forge's publicUrl default follows behindGateway alone, not the central toggle"; - ok = - bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local" - && centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"; - } - { - # And that it still tracks `behindGateway` at all: without this arm the - # case above passes just as well for a default hardcoded to the URL. - name = "the forge's publicUrl default is still null with behindGateway off"; - ok = - (hive { deploy.forgejo.behindGateway = false; }).services.hyperhive.swarm.forge.publicUrl == null; - } - { - # The controller's token path defaulted to forge's delivery path only on - # a host with the central toggle on, and to `null` otherwise. Forge - # deploys unconditionally, so the path is now unconditional too. - name = "the swarm controller's forgeTokenFile defaults to forge's delivery path regardless of the central toggle"; - ok = - let - forgePath = "/var/lib/hyperhive-forge/swarm-controller.token"; - in - bare.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath - && centralToggleOff.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath; - } - { - # `ctl` is in no deny list — it is reserved *because it is the subject a - # cert-auth role accepts*, which is a value an operator sets, so a - # literal deny entry could never have covered it. - name = "a hive named after a cert-auth subject is refused, with the collector off"; - ok = - equalityGuardFired hiveNamedAfterCertSubject - && lib.any (a: !a.assertion && lib.hasInfix "'ctl'" a.message) hiveNamedAfterCertSubject.assertions; - } - { - # Every cert-auth subject is reserved, not just the first one in the - # list. Without this case the second element could be dead and the case - # above would still pass. - name = "a hive named after the secret publisher's subject is refused too"; - ok = - equalityGuardFired hiveNamedAfterPublisherSubject - && lib.any ( - a: !a.assertion && lib.hasInfix "'pubctl'" a.message - ) hiveNamedAfterPublisherSubject.assertions; - } - { - # Without this the case above proves nothing: an arm that fires for every - # roster is not a guard, and `hives` is non-empty in both fixtures. - name = "a legal hive roster trips neither name guard"; - ok = !(equalityGuardFired hiveNamesAllLegal) && !(fragmentGuardFired hiveNamesAllLegal); - } - { - # The substring guard came along in the move and has to still work. - # `h1-agent` mints exactly the client id hive `h1`'s agents present. - name = "a hive name containing a composed-identifier word is refused, with the collector off"; - ok = fragmentGuardFired hiveNameWithComposedWord; - } - { - # ⚠️ The control that makes "with the collector off" mean anything. If a - # fixture silently had swarm-otel enabled, all three cases above would - # pass while testing the arrangement they exist to rule out. - name = "the guard fixtures really do have the collector disabled"; - ok = - !hiveNamedAfterCertSubject.services.hyperhive.deploy.swarm-otel.enable - && !hiveNamesAllLegal.services.hyperhive.deploy.swarm-otel.enable - && !hiveNameWithComposedWord.services.hyperhive.deploy.swarm-otel.enable; - } - { - # `lib.all` over an empty set holds vacuously, so the roster is counted - # before it is read: a helper that lost a member would otherwise turn - # this case green by measuring nothing. - name = "hosting the swarm's shared services turns on every service that switch owns"; - ok = - let - es = swarmServiceEnables swarmServicesHere; - in - lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es); - } - { - name = "a hive that does not host the swarm's shared services runs none of them"; - ok = - let - es = swarmServiceEnables bare; - in - lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es); - } - { - # The switch fills in for an operator who has not spoken and yields to - # one who has — that is what keeps a shared service placeable on a host - # of its own. A plain assignment or `mkForce` would satisfy both cases - # above and break this one. `nats` is the control: without it the case - # also passes on a fixture where nothing came on at all. - name = "placing one shared service elsewhere survives the switch that would enable it"; - ok = - let - es = swarmServiceEnables swarmServicesBaoElsewhere; - in - !es.bao && es.nats; - } - { - # The controller sits on the OTHER tier: `singleHostSwarm` places it - # (./host-modules/local-defaults.nix) and swarm-ui follows the - # controller. Pinned so that moving a service between tiers is a - # decision someone makes rather than a merge nobody reads. - name = "hosting the swarm's shared services does not make a hive the swarm's control plane"; - ok = - !swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable - && !swarmServicesHere.services.hyperhive.deploy.swarm-ui.enable; - } - { - name = "a hive that has not opted into all-local runs no swarm controller"; - ok = !bare.services.hyperhive.deploy.swarm-controller.enable; - } - { - name = "the all-local mode turns the swarm controller on"; - ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; - } - { - # Reads the RENDERED settings, not the option: `calloutBlocks {…} // { - # … }` is a shallow merge, and a future edit that dropped or shadowed - # this key would still evaluate cleanly — the only reader that would - # notice is a publisher whose row exceeds upstream's much smaller - # default, and by then it is a dropped row, not an eval failure. - # Piggybacks on the pre-rename nats fixture above, which already - # renders this container's full config. - name = "the queue's payload ceiling is set, not inherited from the server's default"; - ok = natsOldPath.containers.swarm-nats.config.services.nats.settings.max_payload == 8388608; - } - { - # Reads the RENDERED unit text, not the module's source, because the - # failure this defends against renders perfectly: systemd substitutes - # `$NAME` in `ExecStart` regardless of quoting, so a single dollar - # here hands the responder `.term.{hive}.>` — a grant that parses, is - # accepted, and matches nothing an agent ever publishes to. Asserting - # the doubled dollar is the only way to tell the two apart before - # deploy. The flag's presence is asserted separately so that dropping - # the grant entirely fails as its own arm rather than as an escaping - # complaint. - name = "the responder grants agents their hive's terminal subject, and the dollar survives systemd"; - ok = - let - exec = - natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart; - in - lib.hasInfix "--agent-publish-subject " exec && lib.hasInfix "$$SWARM.term.{hive}.>" exec; - } - { - # Second grant, same escaping trap, asserted separately: the two - # subject families are independent features (terminal rows and the - # turn-state header) and dropping either should fail as its own arm - # rather than being masked by the other still being present. - # - # Flag and argument are matched as one infix rather than as two - # independent `hasInfix` calls: the responder takes the flag - # repeatedly, so the thing worth pinning is that THIS subject is the - # argument of one of them, which two separate presence checks would - # both pass on while the subject sat under some other flag entirely. - name = "the responder grants agents their hive's agent-state subject too"; - ok = - let - exec = - natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart; - in - lib.hasInfix "--agent-publish-subject '$$SWARM.agent-state.{hive}.>'" exec; - } - { - # This fixture enables grafana and NOT authelia, which is the shape the - # login form used to stay enabled in: the toggle read "both services are - # on this host" rather than "grafana requires SSO". Grafana ships an - # `admin`/`admin` account and its vhost is on the public gateway, so a - # password box there is a way in whatever the topology. - name = "grafana disables its local login form even where authelia is not on this host"; - ok = - grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form; - } - { - # The absence class this whole file is for, and the reported defect in one - # arm: the OIDC block hung off "authelia is on this host", so the split - # deployment got a Grafana with no SSO settings and no login form — no way - # in at all. The block is emitted in every deployment now, so the negative - # arm is not "no block elsewhere" but "the two do not name the same IdP": - # each host's block has to point at the URL the SWARM names, and a block - # built from `deploy.authelia` rather than `swarm.authelia.url` would pass - # a presence check on both fixtures while sending one of them nowhere. - name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs"; - ok = - let - oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; - remote = oauth grafanaRemoteAuthelia; - local = oauth grafanaWithAuthelia; - in - remote.enabled - && lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url - && local.enabled - && lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url - && !(lib.hasInfix "auth.example.invalid" local.token_url); - } - { - # 🩸 The arm that guards the ruling this slice landed under. There is ONE - # delivery route: the store reader, on every host that runs Grafana. The - # negative names the deleted unit rather than a generic absence, because - # the way this regresses is someone re-adding the co-located copy as an - # optimisation — a second writer of one path, and a second shape of "the - # secret is wrong" to debug. - name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; - ok = - let - local = grafanaWithAuthelia.systemd.services; - remote = grafanaRemoteAuthelia.systemd.services; - in - local ? swarm-bao-grafana-oidc - && remote ? swarm-bao-grafana-oidc - && !(local ? swarm-grafana-oidc-secret) - && !(remote ? swarm-grafana-oidc-secret); - } - { - # What the deleted warning became. The shape is unchanged — a Grafana host - # holding no store leaf — but silence there is a container nobody can log - # into for a reason no log names, and a warning is read back by nothing. - # The second arm is what makes this a refusal about the IDENTITY: this - # fixture names an IdP, so a message about `swarm.authelia.url` here would - # mean the two assertions had been collapsed into one conjunction. - name = "a grafana host with no store identity is refused, naming the options to set"; - ok = - grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile" - && grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile" - && !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url"); - } - { - # "SSO must always be configured", as an eval-time refusal rather than a - # gate. A null URL used to drop the OIDC block silently, and - # `disable_login_form` is unconditional a hundred lines below it, so that - # combination produced a Grafana with no SSO and no password box — an - # outage whose cause is a boolean that evaluated to false at build time - # and left no trace. Same isolation as the arm above, mirrored. - name = "a grafana host in a swarm with no IdP is refused, naming swarm.authelia.url"; - ok = - grafanaRefusedFor grafanaNoSso "services.hyperhive.swarm.authelia.url" - && !(grafanaRefusedFor grafanaNoSso "deploy.bao.clientCertFile"); - } - { - # Without this the two arms above prove nothing: a refusal that fires on - # every host is not a check, and both of these are hosts a swarm is - # expected to have. Read through the same helper, so a message that - # stopped naming its option would fail the arms above rather than pass - # this one by accident. - name = "neither grafana refusal fires on a correctly configured host, co-located or not"; - ok = - !(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url") - && !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile") - && !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url") - && !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile"); - } - { - # Same 403-not-a-miss reason as the matrix and queue arms below: the - # reader's grant covers the `services` prefix, so a path outside it is - # refused rather than empty, however correct it reads. The negative arm is - # the rename this is exposed to — a secret filed under the hive that runs - # the service instead of under the service itself. - name = "grafana's OIDC secret is read from the prefix the publisher writes"; - ok = - let - s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script; - in - lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s - && !(lib.hasInfix "secret/swarm/hives/" s); - } - { - # Both ends of a wire nothing at eval time carries end to end: the - # publisher on authelia's host writes the path the reader on Grafana's host - # reads, and the two files agree only because both compose it from the same - # swarm-wide client id. - name = "the publisher writes the swarm service path grafana reads"; - ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" ( - secretPublisherHere.systemd.services.swarm-secret-publish.script - ); - } - { - # Both halves of the co-location assumption, which was one host's - # `deploy.*` answering a question about the whole swarm: the identities - # were minted only where the queue happened to run, and the token - # endpoint was known only where the IdP happened to run. - name = "hive identities and the token endpoint do not depend on which host runs what"; - ok = - let - autheliaNoQueue = hive { deploy.authelia.enable = true; }; - in - lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients) - && - grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint - == "https://auth.example.invalid/api/oidc/token"; - } - { - # Registering the client cannot live where the rest of grafana's module - # lives: that block is gated on this host RUNNING grafana, so on the split - # deployment nothing registered the client, authelia minted no secret, and - # every layer below had nothing to carry. The second arm is the control — - # a host with no IdP registers nothing. - name = "the swarm's grafana client is registered wherever authelia runs"; - ok = - let - clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; - in - lib.elem "swarm-grafana" (clients secretPublisherHere) - && !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia)); - } - { - # The collector's half of the same defect and the same fix: this used to - # be gated on `deploy.swarm-otel.enable`, so a split deployment - # registered the client nowhere and authelia minted nothing to publish. - name = "the swarm's collector client is registered wherever authelia runs"; - ok = - let - clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; - in - lib.elem "swarm-collector" (clients secretPublisherHere) - && !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia)); - } - { - # 🩸 The arm that guards the ruling this slice landed under, the - # collector's half of ./swarm-grafana.nix's own. There is ONE delivery - # route: the store reader, on every host that runs the collector and - # holds a store identity. The negative names the deleted unit rather - # than a generic absence, because the way this regresses is someone - # re-adding the co-located copy as an optimisation. - name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; - ok = - let - local = otelBaoWithAuthelia.systemd.services; - remote = otelBaoRemoteAuthelia.systemd.services; - in - local ? swarm-bao-otel-oidc - && remote ? swarm-bao-otel-oidc - && !(local ? swarm-otel-oidc-secret) - && !(remote ? swarm-otel-oidc-secret); - } - { - # Same 403-not-a-miss reason as grafana's arm above: the reader's grant - # covers the `services` prefix, so a path outside it is refused rather - # than empty, however correct it reads. - name = "the collector's OIDC secret is read from the prefix the publisher writes"; - ok = - let - s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script; - in - lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s - && !(lib.hasInfix "secret/swarm/hives/" s); - } - { - # Both ends of a wire nothing at eval time carries end to end: the - # publisher on authelia's host writes the path the reader on the - # collector's host reads, and the two files agree only because both - # compose it from the same swarm-wide client id. `secretPublisherHere` - # already grew this client when `serviceClientIds` did. - name = "the publisher writes the swarm service path the collector reads"; - ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" ( - secretPublisherHere.systemd.services.swarm-secret-publish.script - ); - } - { - # The collector's non-assertion, the deliberate mirror of Grafana's - # assertion two cases up: a host with no store identity is a supported, - # merely degraded shape here, so the reading unit simply does not exist - # rather than refusing the build. `haveCollectorSecret` is what the - # degrade already reads, unchanged by this slice. - name = "a collector with no store identity renders no reading unit, and is not refused"; - ok = - !(otelNoIdentity.systemd.services ? swarm-bao-otel-oidc) - && otelNoIdentity.services.hyperhive.deploy.swarm-otel.clientSecretFile == null - && !(lib.any (a: !a.assertion) otelNoIdentity.assertions); - } - { - # The collector's half of the same split, and a different arm from the - # authenticator case below: this one reads the PATH the unit loads, so a - # reader left on a source that is non-null but wrong still fails. The - # fixture spells the option its pre-rename way, so it covers the rename - # entry at the same time. - name = "a config written against the pre-rename otel secret path still loads it as a credential"; - ok = - lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c) - otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; - } - { - # Reads the rendered unit on the HOST, which is where the write happens: - # every API listener demands a client certificate, and the host is the - # side that has one. - name = "a store host with a placed bootstrap token renders the granting unit on the host"; - ok = - let - u = baoGrantHere.systemd.services.swarm-bao-controller-policy; - in - lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script - && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; - } - { - # The move is the fix, so pin the side it landed on: in the container it - # had no identity to open a connection with, and no address that resolved - # to the store from its own netns. - name = "the granting unit is not rendered inside the store's container"; - ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy); - } - { - # `StartLimit*` are `[Unit]` settings that systemd ignores under - # `[Service]`, so a bound written into `serviceConfig` renders, deploys - # and does nothing. Asserted where nixpkgs puts it rather than where it - # was written. The values are pinned because they are the bound: under - # `shamir` a human unseals by hand, and anything shorter than a day gives - # up first — `start-limit-hit` does not self-heal. - name = "the granting unit's start limit lands in [Unit], not [Service]"; - ok = - let - u = baoGrantHere.systemd.services.swarm-bao-controller-policy; - in - toString u.unitConfig.StartLimitBurst == "2880" - && toString u.unitConfig.StartLimitIntervalSec == "90000" - && !(u.serviceConfig ? StartLimitBurst); - } - { - # The grants themselves, and the `hive-` prefix is the whole point: - # without it the controller can rewrite the policy that constrains it, - # which is a privilege escalation that renders, deploys and looks fine. - # Readable here only because the HCL is piped as an argument rather than - # written to a store path. - name = "the controller's bao grants cannot reach the policy that constrains it"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); - } - { - # Same host-side reasoning as the controller's granting unit above: the - # write needs a client certificate and the host is the side that has one. - name = "a store host with a placed bootstrap token renders the publisher's granting unit too"; - ok = - let - u = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy; - in - u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token" - && lib.hasInfix "swarm-secret-publisher" u.script; - } - { - # The control for the case above, and the same one the controller's unit - # has: rendered on the host means NOT rendered in the container, where it - # would have neither an identity nor a route to the store. - name = "the publisher's granting unit is not rendered inside the store's container"; - ok = - !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); - } - { - # The whole point of a second principal. The two prefixes it publishes to - # and not `swarm/`, so it cannot touch an agent's credentials; and no - # `read`, so a unit whose job is copying a file cannot recover what is - # already there. Pinned as the full capability list per prefix, because an - # added capability is exactly what a presence check misses. - name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; - in - lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s - && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s - && !(lib.hasInfix "secret/data/swarm/agents" s) - && !(lib.hasInfix "secret/data/swarm/*" s) - && !(lib.hasInfix "sys/policies/acl" s); - } - { - # The ordering is load-bearing and invisible at runtime: the controller's - # unit creates the KV and cert-auth mounts this one writes into, so - # without it a cold boot races and fails with "route entry not found", - # which names neither unit. - name = "the publisher's granting unit is ordered after the one that creates the mounts"; - ok = lib.elem "swarm-bao-controller-policy.service" ( - baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.after - ); - } - { - # The policy authorising this route lives in another file, and nothing - # else relates the grants to the paths the code actually writes. - # - # `secret/data/` is KV v2's ACL prefix; `swarm` is - # `swarm_secret_client::path::ROOT` and `agents` is - # `Kind::Agent.as_str()`, both of which that crate pins in its own test. - # - # The grant is still the agent kind alone because nothing writes another - # one yet. It widens when a path outside `agents/` gains a writer, not - # when the kinds are declared. - name = "the controller may write agent credentials, and only under the agent prefix"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "secret/data/swarm/agents/*" s - && !(lib.hasInfix "secret/data/*" s) - && !(lib.hasInfix "path \"secret/*\"" s); - } - { - # Write-only is the property, not an accident of how it was typed: a - # `read` here would let the controller recover every agent's credentials - # instead of only replacing them. Pinned as the whole capability list, - # because an added capability is exactly what a presence check misses. - name = "the controller's grant on agent credentials is write-only"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s; - } - { - # The policy above grants paths under a mount nothing else creates, so - # the unit that writes the policy has to create it too — otherwise every - # certificate login fails against a path that is not there. - name = "the granting unit creates the cert auth mount and the controller's role"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "bao auth enable cert" s - && lib.hasInfix "auth/cert/certs/swarm-controller" s - && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; - } - { - # Same shape as the cert mount above, for the engine the controller - # writes credentials through: a fresh store has no `secret/`, so the - # grant would name a mount nobody created and the first write would 404. - # - # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below - # spells out: the policy text is embedded in this same script and grants - # `secret/data/...`, so any arm keyed on the *path* is satisfied either - # way and could never fail. - name = "the granting unit creates the KV mount the controller writes through"; - ok = - let - s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "bao secrets enable -path=secret kv-v2" s; - } - { - # Nothing asserted the PKI script before this, so a third leaf could be - # added to it and every case still passed — measured, not assumed: the - # commit that added one left `module-eval`'s derivation unchanged. - name = "the store mints a leaf for the controller, and the controller is pointed at it"; - ok = - let - m = baoControllerHere; - pki = m.systemd.services.swarm-bao-pki.script; - in - lib.hasInfix "controller.pem" pki - && - m.services.hyperhive.deploy.swarm-controller.baoClientCertFile - == "/var/lib/swarm-bao-pki/controller.pem" - && - m.services.hyperhive.deploy.swarm-controller.baoClientKeyFile - == "/var/lib/swarm-bao-pki/controller-key.pem"; - } - { - # What makes the one above mean something: a controller with no store - # has nothing to be pointed at. Naming a path here would be a file this - # host never gets, which fails at a TLS handshake rather than at eval. - name = "a controller on a host with no store is left without certificate paths"; - ok = - let - c = controllerNoStore.services.hyperhive.deploy.swarm-controller; - in - c.baoClientCertFile == null && c.baoClientKeyFile == null; - } - { - # Being *pointed at* a leaf and *being handed* one are different claims, - # and the options above were the first without the second — declared, - # defaulted, and read by nothing. This is the arm that makes them reach - # the process. - # - # ⚠️ The LoadCredential source is asserted, not just the `%d` name: the - # controller's leaf and the hive reader's are two identities with two - # policies, and wiring `deploy.bao.clientCertFile` here would satisfy - # every `%d`-only check while giving the daemon a policy that cannot - # write an agent's credential. - name = "the controller is handed its own store leaf, not the hive reader's"; - ok = - let - s = baoControllerHere.systemd.services; - in - s ? swarm-controller - && (s.swarm-controller.environment ? BAO_ADDR) - && (s.swarm-controller.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" - && (s.swarm-controller.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" - && builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential - && builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential; - } - { - # The same hole the controller's case above names, open a second time: the - # PKI script grew a third leaf and no case read it. - name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it"; - ok = - let - m = secretPublisherHere; - p = m.services.hyperhive.deploy.swarm-secret-publisher; - in - lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script - && p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem" - && p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem"; - } - { - # mara caught this by reading, which means no arm existed for it: the - # default asked `authelia.enable && bao.enable`, so the split deployment - # this unit is FOR defaulted off and published nothing, silently. - # - # The second clause is the control. Without it this passes on a default - # of plain `true`, which would be a different bug with the same symptom - # — an IdP-less host claiming it publishes secrets it never mints. - name = "the publisher defaults on where secrets are minted, whether or not the store is local"; - ok = - secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable - && !bare.services.hyperhive.deploy.swarm-secret-publisher.enable; - } - { - # The one security property of this unit, and why its push cannot be - # rewritten into the obvious shape: `bao` is an external binary, so an - # argument is world-readable in /proc for the life of the call. - # `value=@` hands it the path and bao opens the file itself. - # - # The second arm is what makes the first mean anything — `value=@` can - # sit one line above a command substitution that put the plaintext in - # argv anyway. - # - # ⚠️ Comments are stripped first, and that is not tidiness. A `script` - # renders its own comments into the text, and this unit's comments name - # the hazard verbatim so the next editor does not reintroduce it. Without - # the strip this case reads that warning and fails — a check the artifact - # defeats by DESCRIBING the thing it is checked for. - name = "the publisher hands bao the secret's path, never the secret"; - ok = - let - s = secretPublisherHere.systemd.services.swarm-secret-publish.script; - code = lib.concatStringsSep "\n" ( - lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s) - ); - in - lib.hasInfix "value=@" code && !(lib.hasInfix "$(cat" code); - } - { - # Two ends of a wire nothing at eval time carries end to end: this is the - # path `swarm_secret_client::queue` resolves for the reader. Both hives - # are asserted, so a publisher that knew one name rather than the roster - # fails here rather than on the second hive ever added to a swarm. - name = "the publisher writes every hive in the roster to that hive's own queue path"; - ok = - let - s = secretPublisherHere.systemd.services.swarm-secret-publish.script; - in - lib.hasInfix "secret/swarm/hives/h1/queue/agent" s - && lib.hasInfix "secret/swarm/hives/h2/queue/agent" s; - } - { - # The producer's end of the read `glue-matrix-bao-token.nix` already did. - # Both hives are asserted for the reason the queue case above gives: a - # publisher that knew one name rather than the roster would pass on a - # single-hive fixture and strand the second hive ever added — which is - # the two-hives-never-converge shape this slice exists to close. - name = "the publisher mints an appservice token for every hive and writes it to that hive's matrix path"; - ok = - let - s = secretPublisherHere.systemd.services.swarm-secret-publish.script; - in - lib.hasInfix "secret/swarm/hives/h1/matrix/appservice-token" s - && lib.hasInfix "secret/swarm/hives/h2/matrix/appservice-token" s - && lib.hasInfix "/dev/urandom" s; - } - { - # What makes a re-publish idempotent. This principal is granted - # `create`/`update` and no `read`, so it cannot ask the store whether a - # hive already has a token — with nowhere to keep one, every run would - # mint a fresh value and rotate the swarm's token. A state directory is - # that somewhere, and nothing else in this unit needs one, so its absence - # means exactly this. - # - # The second arm is the mint's own guard: the state file is only written - # when it is missing or empty. Dropping that test leaves a unit that - # still has a state directory and still rotates on every boot. - name = "the publisher keeps the tokens it minted, and mints only when it holds none"; - ok = - let - u = secretPublisherHere.systemd.services.swarm-secret-publish; - in - lib.hasInfix "matrix-appservice-token" (u.serviceConfig.StateDirectory or "") - && u.serviceConfig.StateDirectoryMode or null == "0700" - && lib.hasInfix "if [ ! -s \"$src\" ]" u.script; - } - { - # A property of the SET, not of one unit: both of these authenticate by - # certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so - # a script that reaches `bao kv` without a token asks a token helper this - # host does not carry and fails before the store ever answers. `-token-only` - # is what keeps the token off the helper on the way back out. - # - # Ordering, not presence: the login has to come first, so the check is - # that nothing before it is a data command. Comments are stripped because - # both units explain this in prose directly above the code. - name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper"; - ok = - let - code = - s: - lib.concatStringsSep "\n" ( - lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s) - ); - holdsTokenFirst = - s: - let - c = code s; - in - lib.hasInfix "bao login" c - && lib.hasInfix "-token-only" c - && !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c))); - in - holdsTokenFirst secretPublisherHere.systemd.services.swarm-secret-publish.script - && holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-matrix-token.script - # Controls, so a clean verdict above means something. In order: a bare - # read is refused, a read placed before the login is refused, and a - # login that exists only in a comment is refused — that last one is the - # arm the comment-stripping exists for. - && !(holdsTokenFirst "bao kv get -field=value secret/x") - && !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"") - && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") - && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; - } - { - # A login failure is the store being unreachable, sealed, or not yet - # holding this host's role — all of which a retry fixes. A read that - # answers "nothing there" is not, so only the first is allowed to fail - # the unit. - name = "the matrix token reader retries a failed login and still degrades on an empty read"; - ok = - let - u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; - # Everything between the login's failure branch and the read's, which - # is where the exit that decides "retry or give up" lives. - afterLogin = lib.last (lib.splitString "bao login" u.script); - loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); - in - u.serviceConfig.Restart or null == "on-failure" - && u.startLimitBurst or 0 > 0 - # The window has to outlast every attempt, or the burst is unreachable. - && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) - && lib.hasInfix "exit 1" loginBranch - && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); - } - { - # The reader's own grant covers `swarm/hives//*` and - # `swarm/agents/*`; a path outside those answers 403, not "no such key". - # So the hive segment is what makes the read reachable, and a rename that - # drops it looks correct and fails identically on every boot. - name = "the matrix token path sits inside the prefix the reader is granted"; - ok = - let - s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; - in - lib.hasInfix "secret/swarm/hives/" s - && lib.hasInfix "/matrix/appservice-token" s - # The shape it used to have: `matrix` where a principal kind belongs, - # which no grant covers. - && !(lib.hasInfix "secret/swarm/matrix/" s); - } - { - # The store's second reader, and the gate that decides it exists is the - # certificate rather than anything about agents: containers are created - # at runtime, so there is no static "this hive runs agents" fact to ask. - name = "a hive that names a client identity reads its agent queue credential"; - ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent; - } - { - # Absence arm, and what makes the one above able to fail: with no leaf - # this unit would fail a TLS handshake on every boot, so it must not - # exist at all rather than retry its way through the start limit. - name = "a hive with no store identity renders no queue credential reader"; - ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-queue-agent); - } - { - # Same 403-not-a-miss reason as the matrix arm above, against the path - # `swarm_secret_client::queue::agent_client_path` builds from the same - # pieces. The negative arm is the rename this one is exposed to: a - # credential named for the queue rather than for the hive that presents - # it reads as correct and is refused on every boot. - name = "the agent queue credential path sits inside the prefix the reader is granted"; - ok = - let - s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script; - in - lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s); - } - { - # The unit's output is the option's value, not a literal that agrees with - # it today: an operator moving the directory has to move both files. The - # prefix is asserted too because `hasInfix ""` is true — an option - # renamed out from under this arm would otherwise read empty and pass. - name = "the queue credential reader writes both files under the directory its option names"; - ok = - let - m = baoRemoteReader; - dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; - s = m.systemd.services.swarm-bao-queue-agent.script; - in - lib.hasPrefix "/var/lib/" dir - && lib.hasInfix "${dir}/secret" s - && lib.hasInfix "${dir}/client_id" s; - } - { - # A reader off the store's host is a reader whose journal is the only - # record of why a hive's agents never connected, so the collector has to - # be told the unit exists. Nothing else can say it: the store's module - # does not know who holds a certificate. - name = "the queue credential reader's journal reaches the collector"; - ok = builtins.elem "swarm-bao-queue-agent" baoRemoteReader.services.hyperhive.swarm.otel.journaldUnits; - } - { - # No agent container may render before this unit has had its attempts, - # and the edge that guarantees it must delay hive-c0re rather than sink - # it: an unreachable store is this unit's `Restart=on-failure` window, - # not a reason for the daemon that renders every agent to fail its own - # start. - name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it"; - ok = - let - u = baoRemoteReader.systemd.services.swarm-bao-queue-agent; - in - builtins.elem "hive-c0re.service" (u.before or [ ]) - && builtins.elem "hive-c0re.service" (u.wantedBy or [ ]) - && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ])) - && !(builtins.elem "hive-c0re.service" (u.requires or [ ])); - } - { - # Where the reader puts the files and where the daemon looks for them is - # one agreement spanning two modules. Asserted against the option rather - # than the literal so moving the directory moves both ends. - name = "hive-c0re is told where the agents' queue credential lands"; - ok = - allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR - == toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; - } - { - # The one address in this file that must NOT be loopback. Both spellings - # sit in the same unit's environment and are correct for their own - # reader: hive-c0re shares the host netns, an agent container does not, - # so a copy-paste between them reaches the agent itself and the symptom - # is a connect that hangs. - name = "the agents' queue address is the bridge, not the loopback one the hive itself uses"; - ok = - let - e = allLocal.systemd.services.hive-c0re.environment; - in - e.HIVE_AGENT_NATS_URL == "nats://${allLocal.services.hyperhive.network.bridgeIp}:4222" - && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) - && e.HIVE_AGENT_NATS_URL != e.HIVE_C0RE_NATS_URL; - } - { - # The agents mint against the swarm's IdP, the same endpoint the hive's - # own client uses — a hive-local guess would produce a token the queue - # would not accept. - name = "the agents' token endpoint is the swarm IdP's"; - ok = - let - e = allLocal.systemd.services.hive-c0re.environment; - in - lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT - && e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT; - } - { - # The absence arm, and what makes the two above able to fail: a hive - # with no queue address must forward neither coordinate, because half a - # pair reaches the harness as a partial configuration rather than as - # none. - name = "a hive with no swarm queue forwards no agent queue coordinates"; - ok = - let - e = bare.systemd.services.hive-c0re.environment; - in - !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); - } - { - # Both ids or neither: the secret authenticates nobody without the id it - # belongs to, and the harness refuses to treat one of the two as a queue. - name = "an agent with queue coordinates imports both halves of its credential"; - ok = - let - c = (agentHarness agentQueue).serviceConfig.LoadCredential; - in - builtins.elem "hive-queue-agent-secret" c && builtins.elem "hive-queue-agent-client-id" c; - } - { - # `%d` and not a path under the state dir: the host file is `0600` - # root-owned, so the only copy this unprivileged unit can open is the - # one systemd puts in its own credentials directory. - name = "the harness reads its queue credential out of the credentials directory"; - ok = - let - e = (agentHarness agentQueue).environment; - in - e.HIVE_AGENT_OIDC_CLIENT_SECRET_FILE == "%d/hive-queue-agent-secret" - && e.HIVE_AGENT_OIDC_CLIENT_ID_FILE == "%d/hive-queue-agent-client-id"; - } - { - # An agent built before its hive was handed the queue's address. It - # must declare nothing rather than name a credential that never - # arrives — and the harness then reports "no queue coordinates" - # instead of a half-set environment. - name = "an agent with no queue coordinates declares no credential"; - ok = - let - u = agentHarness agentNoQueue; - in - !(u.serviceConfig ? LoadCredential) - && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) - && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); - } - { - # The three ids `hive_c0re::lifecycle::agent_identity` forwards under. - # Neither end can discover the other's spelling, and a mismatch is a - # credential that is simply not there — which this unit then reports as - # a hive that delivered nothing. - name = "an agent with the store enabled imports every half of its identity"; - ok = - let - c = (agentBaoIdentity agentBao).serviceConfig.LoadCredential; - in - builtins.elem "hive-agent-bao-cert" c - && builtins.elem "hive-agent-bao-key" c - && builtins.elem "hive-agent-bao-server-ca" c; - } - { - # `%d` and not a path under the agent's state dir, for the reason the - # queue arm above gives: the host file is `0600` to the hive daemon, so - # the only copy this unprivileged unit can open is the one systemd puts - # in its own credentials directory. The address is the option's value - # rather than a literal that agrees with it today. - name = "the identity check presents its certificate out of the credentials directory"; - ok = - let - e = (agentBaoIdentity agentBao).environment; - in - e.BAO_CLIENT_CERT == "%d/hive-agent-bao-cert" - && e.BAO_CLIENT_KEY == "%d/hive-agent-bao-key" - && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; - } - { - # Same 403-not-a-miss reason as the hive-side readers: the path - # `swarm_secret_client::mtls::identity_path` builds is the one this - # agent's own policy stanza covers, and a path outside it is refused - # however correct it looks. Built from the agent's own name rather than - # from a literal, because the name is what makes it this agent's path - # and not some other agent's. - name = "the identity check reads the agent's own path"; - ok = - let - m = agentBao; - name = m.services.hyperhive.agent.user.name; - in - lib.hasInfix "secret/swarm/agents/${name}/bao-mtls" (agentBaoIdentity m).script; - } - { - # The whole point of the unit, and the thing a quieter default would - # undo: every arm of the check ends the unit non-zero, so an agent that - # cannot authenticate as itself says so at boot instead of at whichever - # pull needed the store first. - name = "the identity check fails the unit rather than degrading"; - ok = - let - u = agentBaoIdentity agentBao; - in - lib.hasInfix "exit 1" u.script - && !(lib.hasInfix "exit 0" u.script) - && u.serviceConfig.Restart == "on-failure"; - } - { - # Nothing about the identity may be printed, and the read-back is where - # that could slip: `bao kv get` on this path answers with certificate - # material, and the object beside it is a private key. The check needs - # only whether the read succeeded. - # - # The path goes through `lib.escapeShellArg` here for the same reason the - # module passes it through one — that helper decides whether an argument - # needs quotes at all, and this one (only `[a-z0-9/-]`) comes back bare. - # Spelling the quotes in by hand asserts a rendering nixpkgs chooses - # rather than the redirect this property is about. - name = "the identity check discards what it reads back"; - ok = - let - m = agentBao; - name = m.services.hyperhive.agent.user.name; - arg = lib.escapeShellArg "secret/swarm/agents/${name}/bao-mtls"; - in - lib.hasInfix "bao kv get -field=cert ${arg} >/dev/null" (agentBaoIdentity m).script; - } - { - # The absence arm, and what makes the four above able to fail. An agent - # whose swarm never minted an identity has nothing to log in with, and a - # failed unit at every boot would be the loudest possible statement - # about a deployment that never asked for one. - name = "an agent told no store address runs no identity check"; - ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity); - } - { - # Where the store is and where the agent is told it is, one agreement - # spanning two modules. Asserted against the hive's own `BAO_ADDR` - # rather than a literal, because an agent pointed at a different - # spelling of the same store presents a certificate to a listener whose - # name it cannot verify. - name = "an agent is told the same store address its hive uses"; - ok = - let - e = allLocal.systemd.services.hive-c0re.environment; - in - e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR; - } - { - # A hive with no certificate of its own can collect no agent's identity, - # so forwarding an address would name a store nothing in the container - # can reach. The same gate the `BAO_*` pair beside it sits behind. - name = "a hive with no store identity forwards no store address to its agents"; - ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR); - } - { - # Two thirds of the container's cap, and a SOFT ceiling: the daemon's - # cgroup holds every nested claude, so `MemoryHigh=` throttles the - # subagent set as a whole before the kernel picks a victim, while the - # absent `MemoryMax=` is what still lets one subagent use more than - # its share on a container that has the memory free. A hard cap here - # would trade the silent kill for a guaranteed wall, which is the - # shape this deliberately does not have. - name = "the subagent daemon throttles at two thirds of the container's memory"; - ok = - let - c = (agentSubagentDaemon agentCapped).serviceConfig; - in - c.MemoryHigh == "5726623061" && !(c ? MemoryMax); - } - { - # What makes the case above able to fail. With no byte count for the - # container there is no fraction to take, and a hardcoded fallback - # would be a number about some other hive's machine — so the unit - # renders no ceiling at all rather than a fabricated one. - name = "an agent with no byte-valued memory cap renders no subagent ceiling"; - ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); - } - { - # The sign is the whole property, and it is easy to write backwards: - # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent - # daemon must be strictly above zero and the harness strictly below - # it — swap the two and the kernel takes the agent's own turn first, - # which is worse than setting nothing at all. Both nested `claude` - # processes inherit their unit's value, so ordering the units orders - # the sessions. Held as an inequality rather than two constants: what - # must not drift is the order, not the magnitudes. - name = "the OOM killer prefers a subagent over the agent's own session"; - ok = - let - sub = (agentSubagentDaemon agentUncapped).serviceConfig.OOMScoreAdjust; - own = (agentHarness agentUncapped).serviceConfig.OOMScoreAdjust; - in - sub > 0 && own < 0 && sub > own; - } - { - # The pair the case above only makes sense with: preferring this unit - # as the victim is an improvement only if losing one subagent isn't - # losing all of them. systemd's default `stop` would take the daemon - # and every sibling session down with whichever process the kernel - # chose, which is the blast radius that made the preference a bad - # trade in the first place. - name = "one subagent losing the OOM draw does not stop the daemon"; - ok = (agentSubagentDaemon agentUncapped).serviceConfig.OOMPolicy == "continue"; - } - { - # The absence class, and a gate asymmetry nothing else can see. A `.path` - # unit is `wantedBy = multi-user.target` and names a `Unit=` by - # convention rather than by reference, so one gated more loosely than the - # service it triggers evaluates clean, deploys clean, and then fails to - # activate the first time the watched file is written. `forge.url`'s own - # option doc already promises the avatar-sync units are "not generated at - # all" without a forge — this is the case that makes that sentence true - # of the watcher and not only of the oneshot. - name = "an agent with an icon and no forge renders neither avatar-sync unit"; - ok = - !(agentIconNoForge.systemd.paths ? forge-avatar-sync) - && !(agentIconNoForge.systemd.services ? forge-avatar-sync); - } - { - # A homeserver URL is the whole input: from it the module derives the - # hive-internal `main` account, and from a non-empty account set the three - # things that used to hang off `matrix.enable`. - name = "an agent with a homeserver gets a main account and the matrix units"; - ok = - let - a = agentMatrix.services.hyperhive.agent.matrixAccounts; - in - lib.attrNames a == [ "main" ] - && a.main.tokenFile == "/agents/a1/state/matrix-token" - && a.main.homeserver == "https://chat.t.local" - && agentMatrix.systemd.services ? hive-matrix-daemon - && agentMatrix.systemd.paths ? hive-matrix-daemon - && agentMatrix.services.hyperhive.agent.extraMcpServers ? matrix; - } - { - # The absence arm, and the reason the enable signal is not vacuous. An - # agent the hive gave no homeserver, whose operator declared nothing, must - # come out with an EMPTY account set — not a `main` that can never log in - # — and therefore with none of the three. Assert the emptiness itself and - # not just the units: it is the account set that is load-bearing now, and - # a `main` sneaking back in is the regression this case exists to name. - name = "an agent with no homeserver and no declared account gets no matrix at all"; - ok = - agentNoMatrix.services.hyperhive.agent.matrixAccounts == { } - && !(agentNoMatrix.systemd.services ? hive-matrix-daemon) - && !(agentNoMatrix.systemd.paths ? hive-matrix-daemon) - && !(agentNoMatrix.services.hyperhive.agent.extraMcpServers ? matrix); - } - { - # Matrix without a hive homeserver: one operator account, its own - # homeserver, no `main`. Under the deleted `matrix.enable` this config was - # an assertion failure ("extras require enable") even though every account - # in it was complete; the account set being the signal is what makes it - # expressible, and the serialized env var is where that has to show up. - name = "an external-only account enables matrix with no main entry"; - ok = - let - accts = agentMatrixExternalOnly.services.hyperhive.agent.matrixAccounts; - env = agentMatrixExternalOnly.systemd.services.hive-matrix-daemon.environment; - in - lib.attrNames accts == [ "ccc" ] - && !(accts ? main) - && - builtins.fromJSON env.HIVE_MATRIX_ACCOUNTS == [ - { - name = "ccc"; - token_file = "/agents/a1/state/matrix-token-ccc"; - state_dir = "/agents/a1/state/matrix-sdk-state-ccc"; - homeserver = "https://matrix.example.invalid"; - } - ] - # No hive homeserver, so nothing may claim one. - && !(env ? HIVE_MATRIX_URL); - } - { - # Case 1 of the additive-merge shape: an agent that declares - # nothing gets exactly the module's base set, no more and no less. - name = "an agent with no claudePlugins definition gets exactly the base set"; - ok = - agentPlugins agentPluginsDefault == [ - "skill-creator@claude-plugins-official" - "base@hyperhive" - ]; - } - { - # Case 2: a plain per-agent definition ADDS to the base set (list-typed - # options at equal priority concatenate) rather than replacing it — - # the property the operator ruling asked for instead of `mkDefault`. - # Sorted before comparing: concatenation order between two same- - # priority definitions is a module-system implementation detail this - # case isn't about — membership and count are. - name = "an agent's own claudePlugins definition adds to the base set"; - ok = - builtins.sort builtins.lessThan (agentPlugins agentPluginsAdded) == [ - "base@hyperhive" - "foo@bar" - "skill-creator@claude-plugins-official" - ]; - } - { - # Case 3: `lib.mkForce` is still the escape hatch — an operator who - # wants the base set gone, not extended, can still say so outright. - name = "an agent's mkForce claudePlugins replaces the base set outright"; - ok = agentPlugins agentPluginsForced == [ "foo@bar" ]; - } - { - # De-dup arm: an agent that names a base-set entry itself must not get - # it installed twice — `lib.unique` at the JSON-render site, not the - # option's merge (the merge is a plain concatenation on purpose, so - # the un-deduped list stays readable for `agentPlugins` above). - name = "an agent repeating a base-set plugin does not get it installed twice"; - ok = - builtins.sort builtins.lessThan ( - builtins.fromJSON agentPluginsDuplicate.environment.etc."hyperhive/claude-plugins.json".text - ) == [ - "base@hyperhive" - "skill-creator@claude-plugins-official" - ]; - } - { - # The doctrine three glue files state, as a property a rewrite has to - # keep: a client is defined by holding a certificate the store accepts, - # never by standing next to the store. Gating this on `deploy.bao.enable` - # would have left the unit rendering only on the one deployment that has - # no use for it. - name = "a publisher holding an identity runs on a host with no store"; - ok = - let - m = secretPublisherRemote; - in - !m.services.hyperhive.deploy.bao.enable - && (m.systemd.services ? swarm-secret-publish) - && (m.systemd.paths ? swarm-secret-publish); - } - { - # What makes the arm above able to fail. Minting the secrets is not being - # able to publish them: with no certificate the unit would fail a TLS - # handshake on every rotation, so it must not exist at all. - name = "an IdP host with no store identity renders no publisher"; - ok = - let - m = secretPublisherNoIdentity; - in - m.services.hyperhive.deploy.authelia.enable - && !(m.systemd.services ? swarm-secret-publish) - && !(m.systemd.paths ? swarm-secret-publish); - } - { - # A hive's cert-auth role carries the authority by value, so the daemon - # has to be handed the file rather than a path into the store's own - # directory it cannot read. - # - # ⚠️ The LoadCredential source is asserted, not just the `%d` name, for - # the reason the arm above gives — and here the wrong file is a - # *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks - # the store's certificate with, evaluates fine in this slot, and would - # make every hive role trust the wrong authority. - name = "the controller is handed the CA hives are issued from"; - ok = - let - s = controllerTwoCas.systemd.services; - m = controllerTwoCas.services.hyperhive; - in - (s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null) - == "%d/hive-client-ca.pem" - && builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential - && !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential - && m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile; - } - { - # Same two-CA fixture, for the same reason: the wrapper verifies the - # STORE, so it takes `serverCaFile`. On a self-signing deployment both - # options name one file and either would pass; here the client CA in that - # slot is a case this arm fails. - # - # ⚠️ The package itself stays off `PATH` — `wrapProgram` renames the real - # binary, so an unwrapped `bao` is unreachable rather than merely - # discouraged. Operator's instruction, and the last assertion is what - # keeps a later "install the package too" from quietly undoing it. - name = "the host gets a wrapped bao CLI"; - ok = baoWrapper != null; - } - { - name = "the wrapped bao CLI carries this store's address"; - ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd; - } - { - # `serverCaFile` and not `clientCaFile`: the wrapper verifies the STORE. - # On a self-signing deployment both options name one file and either - # would pass, which is why this uses the two-CA fixture. - # - # ⚠️ The flag and its VALUE together, escaped the same way the module - # escapes it: `BAO_CACERT` present and `store-server-ca.pem` present - # somewhere are two facts that do not add up to "the CA is set to that - # file", and a weaker pair of `hasInfix`es passes on a wrapper that sets - # neither to the other. - name = "the wrapped bao CLI verifies the store with the server CA"; - ok = - lib.hasInfix "--set-default BAO_CACERT ${lib.escapeShellArg "/etc/pki/store-server-ca.pem"}" baoWrapperCmd - && !lib.hasInfix "hive-clients-ca.pem" baoWrapperCmd; - } - { - # `wrapProgram` renames the real binary, so an unwrapped `bao` is - # unreachable rather than merely discouraged — operator's instruction. - # This is what keeps a later "install the package too" from undoing it. - name = "the unwrapped bao package stays off the host PATH"; - ok = !builtins.elem controllerTwoCas.services.hyperhive.deploy.bao.package baoHostPackages; - } - { - # Absence arm for the one above: without a store identity there is - # nothing to write a role with, so handing over the authority would be - # giving a file to a daemon that cannot act on it. - name = "a controller with no store leaf is given no hive CA either"; - ok = - let - s = controllerNoStore.systemd.services; - in - !(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE) - && !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential); - } - { - # Absence arm for the one above, and what makes it mean anything: a - # controller with no leaf gets no store environment at all rather than - # variables naming files this host never receives. - name = "a controller with no store leaf is given no store environment"; - ok = - let - s = controllerNoStore.systemd.services; - in - s ? swarm-controller - && !(s.swarm-controller.environment ? BAO_ADDR) - && !(lib.any (c: lib.hasPrefix "bao-" c) s.swarm-controller.serviceConfig.LoadCredential); - } - { - # The CN is an interface between two files: the store writes a role that - # matches it, the PKI mints a leaf that carries it. They read one option, - # and this is what says so — the fixture's value cannot come from a - # default, so matching it in both places is not a coincidence. - name = "the cert-auth role and the minted leaf take their subject from one option"; - ok = - let - m = baoControllerHere; - role = m.systemd.services.swarm-bao-controller-policy.script; - pki = m.systemd.services.swarm-bao-pki.script; - in - lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki; - } - { - # The arm that makes the one above mean something. A role's trust anchor - # is the CA, so with none named there is nothing to write — and the - # policy write, which needs no CA, must survive that. - # - # ⚠️ Matched on the COMMANDS, not on `auth/cert/certs`: the policy text is - # embedded in this same script and grants that very path, so the shorter - # infix is present either way and the arm could never fail. - name = "with no client CA the unit still writes the policy and skips the role"; - ok = - let - s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; - in - lib.hasInfix "bao policy write" s - && !(lib.hasInfix "bao auth enable cert" s) - && !(lib.hasInfix "client-ca.pem" s) - # The KV mount is NOT part of what a missing client CA switches off: - # the controller writes through it whether or not anything can log in - # by certificate. Asserted here rather than trusted, because both - # steps live in the same script and one indentation level decides it. - && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; - } - { - # What makes the granting-unit cases mean something, and the property - # the host-side half depends on: no store here, so no bind mount and no - # unit. Without it a hive that merely names a token would drag the - # store's container config into its evaluation. - name = "a bootstrap token on a host that runs no store grants nothing"; - ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); - } - { - # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture - # cannot define it; what can break is a reader left pointing at the - # namespace it moved out of. Five modules read this through an - # `autheliaCfg` alias, where a path-shaped grep does not see it. - name = "a consumer of authelia's host client-secret dir renders it from the deploy namespace"; - ok = lib.hasInfix "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients/" autheliaOldPath.systemd.services.swarm-nats-auth-secrets.script; - } - { - # The gateway's per-name issuer choice. If this ever collapses to a - # constant, every swarm-service vhost serves a certificate its CA - # is name-constrained out of — which evaluates cleanly and fails in - # a browser. - name = "a swarm service name gets the swarm-services leaf and the default server does not"; - ok = - let - l = allLocal.services.hyperhive.gateway.lib; - in - (l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate; - } - { - # nixos asserts when a vhost declares both, so this is also a - # statement that the `removeAttrs` upstream of it still happens. - name = "the swarm UI vhost forces TLS instead of merely adding it"; - ok = - let - v = allLocal.services.nginx.virtualHosts."t.local"; - in - v.forceSSL && !(v.addSSL or false); - } - { - name = "a hive with matrix off serves no matrix discovery endpoint"; - ok = - !(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations); - } - { - # main got eval-borked twice by this exact class of bug (once on the - # unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to - # `gitea-actions-runner.nix` adds a plain `serviceConfig.*` - # definition that collides with one of ours, and nix refuses to - # merge two plain definitions at *host* eval. No other check - # instantiates a host with `containers.hive-ci` actually enabled, so - # the collision only surfaces on operator deploy, not in CI. - name = "the CI container's unit definitions merge without a priority collision"; - ok = forceCiServiceConfigs; - } - { - # The defect itself. These exporters used to be gated on the stores' - # PER-HOST enables, so a collector that did not share a host with them - # rendered none at all and dropped everything it received, from every - # hive — silently, because an absent exporter is not an error. - name = "a collector that hosts neither store still exports to both"; - ok = - let - e = (otelSettings otelNoStores).exporters; - in - (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs"); - } - { - # A swarm has one of each store, so the address is a swarm-level name. - # A loopback literal here is the co-location assumption written back in, - # and it renders, deploys and reports healthy while reaching nothing. - name = "the store exporters address the stores by name, never by loopback"; - ok = - let - e = (otelSettings otelNoStores).exporters; - m = e."otlphttp/victoriametrics".metrics_endpoint; - l = e."otlphttp/victorialogs".logs_endpoint; - in - !(lib.hasInfix "127.0.0.1" m) - && !(lib.hasInfix "127.0.0.1" l) - && lib.hasInfix "metrics.t.local" m - && lib.hasInfix "logs.t.local" l; - } - { - # `_HOSTNAME` cannot separate machines on its own: a hostname is a - # config value two of them can share, and then every stream for a unit - # name merges into one. - name = "the log stream is keyed by machine, not only by a hostname every container shares"; - ok = - let - l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; - field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; - in - field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); - } - { - # The collector reaches these routes through the gateway now, so each - # store needs an ingest location of its own. Without one the write rides - # the `/` catch-all: unauthenticated on the metrics store, and into a - # browser redirect on the log store. - name = "each store's vhost has an authenticated ingest location"; - ok = - let - v = allLocal.services.nginx.virtualHosts; - m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null; - l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null; - in - m != null - && l != null - && lib.hasInfix "auth_request" m.extraConfig - && lib.hasInfix "auth_request" l.extraConfig; - } - { - # Two limits governed a matrix upload and nothing kept them in agreement, - # so raising the documented one past the gateway's hardcoded cap changed - # nothing. The second clause is the control: the old directive is gone, so - # a pass means the value is derived rather than that `hasInfix` matched - # something incidental. It targets the whole directive rather than the - # bare size, because the rendered config carries the comment above it and - # a prose mention of the old value would defeat a looser arm. - name = "the matrix gateway's body cap is derived from maxRequestSize, not a literal"; - ok = - let - c = matrixBodyCap.services.nginx.virtualHosts."chat.t.local".locations."/_matrix/".extraConfig; - in - lib.hasInfix "client_max_body_size 100048576;" c && !(lib.hasInfix "client_max_body_size 50M" c); - } - { - # The arm that actually protects something. A pusher handed - # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page, - # which answers 200 — ingest reporting healthy while storing nothing. - # The third clause is the positive control: the log store's browser - # location really does redirect, so this says the machine routes differ - # rather than that the string is absent from the whole file. - name = "the ingest locations answer 401 instead of redirecting a pusher"; - ok = - let - v = allLocal.services.nginx.virtualHosts; - m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig; - l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig; - browser = v."logs.t.local".locations."/".extraConfig; - in - !(lib.hasInfix "error_page" m) - && !(lib.hasInfix "error_page" l) - && lib.hasInfix "error_page" browser; - } - { - # The read counterpart to the ingest location: an agent queries the log - # store with a bearer token, and the `/` catch-all is the browser's - # route. Riding it would mean inheriting the login redirect the next - # case is about, so the route has to exist separately to be gated - # separately. - name = "the log store's vhost has an authenticated machine query location"; - ok = - let - q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null; - in - q != null && lib.hasInfix "auth_request" q.extraConfig; - } - { - # Same trap as the ingest case, on the read side, where it is worse: a - # redirected pusher at least stores nothing visibly, while a redirected - # *reader* is handed a 200 carrying login HTML and records a query that - # succeeded and matched no logs. The browser clause is the positive - # control — that location really does redirect to the login host — so a - # pass means these two routes differ rather than that the strings are - # absent from the whole vhost. - name = "the machine query location answers 401 instead of redirecting to a login page"; - ok = - let - v = allLocal.services.nginx.virtualHosts; - q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig; - browser = v."logs.t.local".locations."/".extraConfig; - in - !(lib.hasInfix "error_page" q) - && !(lib.hasInfix "auth.t.local" q) - && lib.hasInfix "error_page" browser - && lib.hasInfix "auth.t.local" browser; - } - { - # Read access is deliberately unscoped: an authenticated caller reads - # the whole swarm's logs until a permission system exists. Pinned so a - # scoping parameter arriving later is a visible diff here rather than a - # quiet change of rule — and pinned on `proxyPass` too, because - # VictoriaLogs takes its filters as request parameters, which ride an - # upstream URI as easily as a directive. The first clause is the - # control: it proves the location resolved and that `hasInfix` finds - # what is genuinely in this string, so the absences below mean absent - # rather than unreadable. - name = "the machine query location forwards the caller's query unmodified"; - ok = - let - q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/"; - in - lib.hasInfix "auth_request" q.extraConfig - && !(lib.hasInfix "extra_filters" q.extraConfig) - && !(lib.hasInfix "extra_stream_filters" q.extraConfig) - && !(lib.hasInfix "$args" q.extraConfig) - && !(lib.hasInfix "?" q.proxyPass); - } - { - # Defining an exporter and REFERENCING it are two separate lists, and - # the second is where the original gate also lived. An exporter no - # pipeline names is as silent as one that does not exist — this case - # exists because a mutation that restored only the reference-side gate - # left every other case here green. - name = "every pipeline that has a store exporter defined actually sends to it"; - ok = - let - s = otelSettings otelNoStores; - used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines)); - in - builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used; - } - { - # The collector authenticates because it HOLDS a credential, not because - # authelia happens to share its host. Gating on the other service's - # placement renders a collector that pushes unauthenticated wherever - # authelia lives elsewhere — one of the supported shapes. - name = "a collector with a hand-delivered secret authenticates without authelia beside it"; - ok = - let - s = otelSettings otelRemoteAuthelia; - in - (s.exporters."otlphttp/victoriametrics" ? auth) - && builtins.elem "oauth2client/victoriametrics" s.service.extensions; - } - { - # An authenticator an exporter names but `service.extensions` omits is - # INERT — the collector starts clean and pushes unauthenticated until - # something at the far end refuses it. Checked as a set relation rather - # than by naming the two, so it keeps holding for exporters not written - # yet. - name = "every exporter authenticator is listed in service.extensions"; - ok = - let - s = otelSettings otelNoStores; - named = lib.filter (v: v != null) ( - lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters - ); - in - named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; - } - { - # The journald receiver's own default directory is the RUNTIME - # journal, and a container that stores persistently leaves that - # empty. At the default the forwarder validates, starts, reports - # healthy and ships nothing, so this one literal is the difference - # between the path working and silently not. - name = "the agent forwarder reads the persistent journal, not the runtime one"; - ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; - } - { - # The hop's two ends: what it reads, and where what it reads goes. - # The endpoint is compared against the value the fixture handed the - # option rather than a literal spelled here, so an exporter that - # stopped reading the option fails instead of matching a constant - # that travelled beside it. - name = "the agent forwarder ships the journal to the endpoint its hive gave it"; - ok = - let - s = agentSettings agentOtel; - p = s.service.pipelines.logs; - in - p.receivers == [ "journald" ] - && p.exporters != [ ] - && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; - } - { - # Presence control for the two cases above: with the switch off there - # is no collector in the container at all, so their passing is about - # the wiring rather than about a unit that renders regardless. - name = "an agent that has not opted into telemetry runs no collector"; - ok = !agentNoOtel.services.opentelemetry-collector.enable; - } - { - # `otlp` and `otlphttp` are different components and the protocol - # option picks which one is defined. A pipeline left naming the other - # is a startup failure; an exporter no pipeline names is silence. - name = "the agent forwarder's exporter and its pipeline agree on the protocol"; - ok = - let - s = agentSettings agentOtelGrpc; - in - (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; - } - { - # The tier in the middle. Its OTLP receiver takes both signals on one - # port, so without this pipeline an agent's push is answered 404 on - # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish- - # able from one with nothing to send. Compared against the metrics - # pipeline's exporters rather than a name, so the two signals cannot - # drift to different destinations. - name = "the hive collector forwards logs upstream, not only metrics"; - ok = - (hiveOtelPipelines ? logs) - && hiveOtelPipelines.metrics.exporters != [ ] - && hiveOtelPipelines.logs.receivers == [ "otlp" ] - && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters; - } - { - # `deltatocumulative` is metrics-only: naming it in a logs pipeline - # kills the collector at startup rather than doing nothing. The second - # clause is the control — the metrics pipeline still names it, so a - # pass means the two processor lists differ rather than that the - # processor left the module. - name = "the hive collector keeps the metrics-only processor out of its logs pipeline"; - ok = - !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) - && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; - } - { - # The counters that say telemetry is being LOST — refused, failed, - # queue depth — are served on loopback and reach no store unless - # something reads them. Read off the rendered job rather than the - # option: only the job is what prometheus actually requests. - name = "the hive collector scrapes its own telemetry endpoint"; - ok = - let - j = hiveScrapeJob "collector"; - in - j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ]; - } - { - # A `prometheus` receiver no pipeline names collects nothing while - # rendering and starting perfectly, so the scrape above is inert - # without this. The path is newly reachable: until the hive tier had a - # target of its own, this receiver was never emitted on any hive. - name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs"; - ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers; - } - { - # `metrics.address` is the spelling that looks right and is rejected by - # this collector version. The first clause is the control: without it a - # missing telemetry block would pass the port check vacuously. - name = "the hive collector binds its telemetry port through readers, not address"; - ok = - let - m = hiveOtelSettings.service.telemetry.metrics; - in - !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; - } - { - # Same gap one tier up, and it needs its own arm: this collector - # already had six scrape targets, so a pass here is about the seventh - # rather than about the receiver existing at all. - name = "the swarm collector scrapes its own telemetry endpoint"; - ok = - let - j = scrapeJob baoWithCollector "collector"; - in - j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ]; - } - { - # Both collectors share a network namespace whenever they are - # co-located, and this port appears in no config the port-collision - # assertion can read — so equal defaults mean the second to start dies - # at `bind()`. Pinned as a case rather than an assertion: enforcing it - # belongs with the other port checks, not here. - name = "the two collector tiers do not claim the same self-telemetry port"; - ok = - let - portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port; - in - portOf hiveOtelSettings != portOf (otelSettings baoWithCollector); - } - { - # The absence arm for the case above, and the option's own rule — a - # service declares its entry under its own `enable` — made checkable. - # Without it, moving the assignment outside the collector's `mkIf` - # passes every arm above while handing a collector-less hive a scrape - # target for a port nothing binds. - name = "a hive with no collector declares no self-scrape target"; - ok = bare.services.hyperhive.otel.scrapeTargets == { }; - } - { - # Defining a receiver and attaching it are two separate lists, and the - # two gates were spelled differently: the receiver appeared for either - # scrape option, the pipeline only for the loopback one. A published- - # only collector therefore rendered scrape configs that reached no - # pipeline — requested, parsed, delivered nowhere, and valid enough to - # deploy. The receiver clause is what stops the arm passing for the - # wrong reason, by an empty `prometheus` never rendering at all. - name = "a published-only collector attaches its prometheus receiver to the swarm pipeline"; - ok = - let - s = otelSettings otelOnlyPublished; - in - otelOnlyPublished.services.hyperhive.swarm.otel.scrapeTargets == { } - && otelOnlyPublished.services.hyperhive.swarm.otel.publishedScrapeTargets != { } - && (s.receivers ? prometheus) - && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; - } - { - # Read against the roster the fixture declares rather than against - # names spelled here: an arm naming `h1` passes on a single-hive - # config however the mapping is written. The length clause is what - # makes the `all` mean anything — over an empty roster it holds - # vacuously. - name = "the swarm collector routes every hive's logs, not just one"; - ok = - let - p = (otelSettings otelTwoHives).service.pipelines; - hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives; - in - lib.length hives == 2 - && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives; - } - { - # The same split as the metrics case above — defining an exporter and - # naming it are two lists — plus the half one shared list cannot have: - # the metrics store's exporter renders perfectly well inside a logs - # pipeline and posts journal records at an ingest route that is not - # for them. - name = "every logs pipeline sends to the log store and to no metrics one"; - ok = - let - s = otelSettings otelTwoHives; - logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines; - used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes)); - in - logPipes != { } - && builtins.elem "otlphttp/victorialogs" used - && !(builtins.elem "otlphttp/victoriametrics" used) - && lib.all (e: s.exporters ? ${e}) used; - } - { - # The store's seal is spread over six gates — the stanza, the - # provisioning unit, two bind mounts, a device and an EnvironmentFile. - # Rendering only some of them is the dangerous state: a store that - # says hardware-backed and seals with a software key, which no - # assertion can catch because every value is individually valid. - name = "a shamir store renders no TPM provisioning unit"; - ok = !(baoUnits baoShamir ? swarm-bao-token); - } - { - # Presence control for the case above. Without it, a typo in the - # option name would satisfy the absence arm forever. The second half is - # the fix itself: the unit has to run where openbao's `DynamicUser` is - # allocated, and a host unit writing the same bytes has no name to hand - # them to. - name = "a pkcs11 store provisions the token in the container, not on the host"; - ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token); - } - { - # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn - # mounts its own /dev and cannot create device nodes, so permission to - # use a device that was never bound in opens nothing. Neither half - # fails on its own, which is why they are asserted as a pair. - name = "a pkcs11 store gets the TPM device bound in, not merely allowed"; - ok = - let - c = baoPkcs11.containers.swarm-bao; - in - (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices; - } - { - # The stanza's label and the label the unit creates are two literals that - # have to name one object, and the mechanism is asserted with them - # because it is valid only for an RSA key: openbao takes AES-GCM or - # RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the - # seal asked for. Every wrong combination renders and deploys, and - # surfaces as a pkcs11 error at `operator init`. - name = "the seal asks for the RSA key the provisioning unit creates"; - ok = - let - p = (baoSettings baoPkcs11).seal.pkcs11; - in - (p.mechanism or "") == "CKM_RSA_PKCS_OAEP" - && lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" ( - (baoUnits baoPkcs11).swarm-bao-token.script or "" - ); - } - { - # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is - # read-only to the seal however it is owned, and the group is the only - # handle on a uid allocated at start. Dropping either surfaces as a - # pkcs11 error deep in a library, naming neither the mount nor the user. - name = "the store's seal may write the token directory, and is in both its groups"; - ok = - let - sc = (baoUnits baoPkcs11).openbao.serviceConfig; - in - # `or [ ]` rather than a bare select: the interesting mutation is the - # key being gone, and a select would abort the whole run with a nix - # trace instead of failing this case by name. - builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) - && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]) - && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]); - } - { - # The device node belongs to the HOST and is matched by NUMBER, while the - # unit that opens it lives in the container — so the two sides holding - # the same gid is the entire mechanism. Letting either side auto-allocate - # renders cleanly, deploys cleanly, and leaves a 0660 node the seal - # cannot open. Compared rather than each checked against a literal: the - # property is that they AGREE, not what they agree on. - name = "the TPM group has the same gid on the host and inside the container"; - ok = - let - host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null; - inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null; - in - host != null && host == inner; - } - { - # Absence arm for the case above — a shamir store never opens a TPM, so - # it must not claim a device node's group. Without this, pinning the gid - # unconditionally would look identical. - name = "a shamir store claims no TPM device group"; - ok = !(baoShamir.users.groups ? swarm-bao-tpm); - } - { - # The store's mTLS identity is a separate trust domain from both CAs in - # this tree, because it must not come from an authority the store will - # itself distribute. What supplies it is the glue, which mints a CA of - # the store's own — so an enabled store has all three paths, and if this - # ever reads null again the store stops coming up on its own. - name = "a deployed store is given its own certificate, key and client CA"; - ok = - let - b = baoPkcs11.services.hyperhive.deploy.bao; - in - b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null; - } - { - # Everything the glue sets is `mkDefault`, and this is the case that - # says so: a deployment whose certificates come from somewhere the glue - # has never heard of must win. Also the presence control for the case - # above — a renamed option would read `null` on both and satisfy - # neither, but only this one names a value. - name = "an operator's own certificate path beats the glue's default"; - ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem"; - } - { - # The store's first reader. Its unit belongs to the pairing, not to - # either service: matrix must not learn the store exists, and the store - # must not know who reads it. - name = "a store deployed beside the homeserver fetches its appservice token"; - ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token; - } - { - # Absence arm. A store with nothing to serve renders no reader, so the - # unit is a function of the PAIRING rather than of the store — which is - # the property that makes it glue instead of a feature of either side. - name = "a store with no homeserver beside it renders no token reader"; - ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token); - } - { - name = "a hive that names a client identity reads from a store it does not run"; - ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token; - } - { - # Absence arm for the one above, and the reason the gate is the identity - # rather than the homeserver: without it, deploying matrix anywhere would - # render a reader that cannot authenticate. - name = "a homeserver with no way to authenticate to the store renders no token reader"; - ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-matrix-token); - } - { - # `Requires=` on a unit that does not exist fails the job, and nothing - # local mints certificates off-host — so this orders against nothing. - # Eval cannot see that failure; only the empty list here stands in for it. - name = "an off-host reader requires no unit the store's host would have provided"; - # Membership first, then the value: indexing a missing unit throws, and a - # table that reports which property broke must not be the thing that dies. - ok = - let - s = baoRemoteReader.systemd.services; - in - s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ]; - } - { - # Presence control for the case above: the list is conditional, not gone. - name = "a co-located reader still orders after the local pki unit"; - ok = - let - s = baoWithMatrix.systemd.services; - in - s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ]; - } - { - # Nothing asserted this script before, which is how it kept a branch that - # named three states and threw away the only thing telling them apart. A - # missing value, a refused identity and an unreachable host all end in the - # same degraded mode here, correctly — what must survive is which one. - name = "the matrix token reader carries the store's own diagnostic into the journal"; - ok = - let - s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; - in - !(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s; - } - { - # hive-c0re runs as hive-core and the client key is `0600` root-owned - # inside a `0700` directory, so the identity reaches the daemon as a - # systemd credential and the environment names `%d` rather than the - # file. Both halves are asserted together because either alone is a - # daemon that fails at the TLS handshake, naming neither. - name = "a reader hands hive-c0re a store identity the daemon cannot open itself"; - ok = - let - s = baoRemoteReader.systemd.services; - in - s ? hive-c0re - && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" - && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" - && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential - && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential; - } - { - # Absence arm for the case above. A hive with no client identity gets no - # store environment at all — the daemon reports a queue it cannot serve - # rather than a handshake it cannot explain. - name = "a hive with no client identity gives hive-c0re no store environment"; - ok = - let - s = matrixNoBaoIdentity.systemd.services; - in - s ? hive-c0re - && !(s.hive-c0re.environment ? BAO_ADDR) - && !(lib.any (c: lib.hasPrefix "bao-" c) s.hive-c0re.serviceConfig.LoadCredential); - } - { - # The CA is its own arm: absent means the system trust store, which is - # right for a deployment with a real CA and wrong for a self-signed one. - name = "a reader that names no store CA falls through to the system trust store"; - ok = - let - s = baoRemoteReader.systemd.services; - in - s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT); - } - { - # Presence control for the arm above: the CA is conditional, not gone. - # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it. - name = "a reader beside a self-signed store is given that store's CA"; - ok = - let - s = baoWithMatrix.systemd.services; - in - s ? hive-c0re - && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem" - && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential; - } - { - # The name a reader dials has to resolve where the store runs; a - # multi-host swarm resolves it upstream instead. - name = "the store's host answers for the store's name"; - ok = builtins.elem "bao.t.local" (baoNames baoPkcs11); - } - { - # Absence arm, and the one that matters: claiming a name this host does - # not serve points every local reader at the wrong machine. - name = "a hive that does not run the store claims no name for it"; - ok = !(builtins.elem "bao.t.local" (baoNames bare)); - } - { - # What makes that name reachable from inside an agent container, and the - # single reason it is a stream server rather than a vhost: `ssl_preread` - # routes on the SNI without decrypting, so the handshake bao completes is - # still the client's own and the certificate it authenticates by arrives - # intact. Terminating here would hand the store one identity for the - # whole swarm. - name = "the store's host passes connections through without terminating TLS"; - ok = - let - s = baoStream baoPkcs11; - in - lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s; - } - { - # The address half, and it is bridge-only for a reason a wildcard would - # hide until deploy: the store already holds `127.0.0.1:8200` in this - # same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start - # — taking every hive domain behind the gateway down with it. - name = "the passthrough listens on the bridge, not on every address"; - ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11); - } - { - # The routing half: the SNI picks the backend and the only name that - # resolves to one is the store's own. A `default` that pointed anywhere - # would make this host a relay for whatever name a client invented. - name = "the passthrough routes only the store's name, to its loopback listener"; - ok = - let - s = baoStream baoPkcs11; - in - lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s - && lib.hasInfix "bao.t.local 127.0.0.1:8200;" s - && lib.hasInfix ''default "";'' s; - } - { - # ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a - # host-wide option, so a block rendered outside the store's own `mkIf` - # gives every hive in the swarm a listener — on the port the store - # answers on, in front of no store at all. - name = "a hive that does not run the store renders no stream passthrough"; - ok = baoStream bare == ""; - } - { - # The listener is only half of reachable: the bridge firewall drops - # everything not named here, and a silent drop is the failure that reads - # as "the store is down" from inside a container. - name = "the store's port is open on the bridge where the store runs"; - ok = builtins.elem 8200 (bridgePorts baoPkcs11); - } - { - # Absence arm for the case above — a hive with no store has no reason to - # open the store's port, and opening it would point agents at a host that - # answers nothing. - name = "a hive that does not run the store opens no bridge port for it"; - ok = !(builtins.elem 8200 (bridgePorts bare)); - } - { - # The store stays behind the passthrough rather than beside it: loopback - # plus whatever was declared, never the bridge. A store that also bound - # the bridge itself would collide with the listener above, and the - # colliding one is nginx — the whole gateway, not just this port. - name = "the store binds loopback and its declared addresses, never the bridge"; - ok = - let - l = (baoSettings baoTwoAddresses).listener; - in - l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200"; - } - { - # Raft refuses to start without it, and says so in a message that names - # neither the setting nor the stanza. - name = "the store advertises a cluster address"; - ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or ""); - } - { - # Control for the case above: these settings are rendered per deployment, - # not constants a passing case could be indifferent to. - name = "a declared extra address renders a second listener beside loopback"; - ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; - } - { - # Retention is what serves the endpoint at all, so the listener alone - # would be a port that answers 404. - name = "a store beside a collector serves metrics on its own listener"; - ok = - let - s = baoSettings baoWithCollector; - in - s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; - } - { - # openbao serves no `/metrics` at all, so a scrape of the default path - # 404s: the store looks like a dead exporter, and every panel built on - # it renders empty rather than erroring. - name = "the store's scrape asks for the path openbao serves"; - ok = - let - j = scrapeJob baoWithCollector "bao"; - in - (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; - } - { - # Presence control for the case above: both fields are omitted rather - # than defaulted, so a target declared as bare `host:port` renders what - # it rendered before the path grammar existed. - name = "a target with no path renders neither metrics_path nor params"; - ok = - let - j = scrapeJob baoWithCollector "plain"; - in - j != null && !(j ? metrics_path) && !(j ? params); - } - { - # Absence arm. Unauthenticated by design, so it must not exist where - # nothing reads it. - name = "a store with no collector beside it serves no metrics"; - ok = - let - s = baoSettings baoNoCollector; - in - !(s.listener ? metrics) && !(s ? telemetry); - } - { - # The four blocks below used to be gated on the hive being enabled AND - # their own condition. The second half was always the load-bearing one — - # none of these conditions is derived from the hive toggle — so these - # arms pin what the conjunct was doing: nothing. Each is written against - # a host with the hive OFF as well as one with it on, because the way a - # dropped conjunct fails is by making something unconditional, and that - # shows up as a service appearing where nothing asked for it. - name = "no bridge port is opened for an exposeHostPorts nobody set"; - ok = - !(builtins.elem 5432 (bridgePortsOrNone bare)) - && !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff)); - } - { - # Control for the arm above, and the one place this slice is not inert: - # the ports are opened because they were named, on a host that never - # turned the hive on. The bridge firewall is the host's own, so there is - # nothing here for the hive toggle to have been protecting. - name = "a named exposeHostPorts opens its bridge port on the host's own say-so"; - ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive); - } - { - # `deploy.swarm-controller.enable`, which defaults false and is - # deliberately not derived from the hive toggle — a swarm has one - # controller, so the host that runs it says so itself. - # - # Probed by what the daemon needs in order to run, not by - # `? swarm-controller`: ./host-modules/hive-tls.nix defines an - # environment key on that unit name, which leaves the attr - # present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive - # that has a CA — see the comment there. The credential oneshot has no - # second definer, so its absence is the unambiguous half. - name = "the swarm controller does not run unless this host is told to run it"; - ok = - let - inert = - machine: - !(machine.systemd.services ? swarm-controller-credential) - && !( - (machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart - ); - in - inert bare && inert centralToggleOff; - } - { - # `deploy.swarm-otel.enable`, same shape: the swarm's collector is one - # host's job, and the container is the whole of what it renders. - name = "the swarm collector container is absent unless this host is told to run it"; - ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel); - } - { - # `deploy.swarm-ui.enable`, which defaults to the controller's toggle — - # derived from a sibling deployment decision, still not from the hive - # toggle. `t.local` is the fixtures' swarm domain, which is the apex the - # UI claims; the arm below is what proves this vhost renders at all. - name = "the swarm UI vhost is absent unless this host is told to serve it"; - ok = - !(bare.services.nginx.virtualHosts ? "t.local") - && !(centralToggleOff.services.nginx.virtualHosts ? "t.local"); - } - { - name = "the swarm UI claims the swarm apex where this host serves it"; - ok = swarmUiHere.services.nginx.virtualHosts ? "t.local"; - } - { - # The three infrastructure toggles are off by default and asserted by - # whoever needs them. With nothing on the host needing them, none of - # the three renders — which is also the control for the arm below. - name = "the gateway, resolver and bridge are absent where nothing on the host needs them"; - ok = - !centralToggleOff.services.hyperhive.gateway.enable - && !centralToggleOff.services.hyperhive.gateway.dns.enable - && !centralToggleOff.services.hyperhive.network.enable - && !(centralToggleOff.services.nginx.enable or false) - && !(centralToggleOff.services.dnsmasq.enable or false) - && !(centralToggleOff.networking.bridges ? hive-br0); - } - { - # hive-c0re asserts all three, and it follows the central toggle — so - # an ordinary hive keeps getting them with no opt-in, which is what - # this change must not break. - name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them"; - ok = - bare.services.hyperhive.gateway.enable - && bare.services.hyperhive.gateway.dns.enable - && bare.services.hyperhive.network.enable - && bare.services.nginx.enable - && bare.services.dnsmasq.enable - && bare.networking.bridges ? hive-br0; - } - { - # The swarm-services toggle enables them explicitly, from its own - # module rather than from any of their defaults. - name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself"; - ok = - swarmServicesOnly.services.hyperhive.gateway.enable - && swarmServicesOnly.services.hyperhive.gateway.dns.enable - && swarmServicesOnly.services.hyperhive.network.enable; - } - { - # An operator's explicit `false` beats every `mkDefault` assertion, - # which is what keeps "asserted by whoever needs it" from being a - # setting the operator cannot turn off. - name = "an explicit gateway.enable = false wins over the modules asserting it"; - ok = !(hive { gateway.enable = false; }).services.nginx.enable; - } - ]; - - bad = builtins.filter (c: !c.ok) cases; - # Escaped, because a name is prose and prose contains apostrophes. Hand-quoting - # broke the builder mid-report on the first such name that failed — and only - # ever on failure, so every green run agreed the reporter was fine. - report = lib.concatMapStringsSep "\n" ( - c: " echo ${lib.escapeShellArg "FAILED: ${c.name}"} >&2" - ) bad; -in -# The results are embedded in the builder text on purpose: that is what -# makes this derivation's hash depend on them, so a nix-only change that -# flips a case cannot be answered from cache. -pkgs.runCommand "hyperhive-module-eval" { } '' - ${report} - ${ - if bad == [ ] then - "echo '${toString (builtins.length cases)} module properties hold' && touch $out" - else - "echo 'module-eval: ${toString (builtins.length bad)} of ${toString (builtins.length cases)} properties broke' >&2 && exit 1" - } -'' diff --git a/nix/module-eval/agent-icon.nix b/nix/module-eval/agent-icon.nix new file mode 100644 index 00000000..f8843064 --- /dev/null +++ b/nix/module-eval/agent-icon.nix @@ -0,0 +1,49 @@ +# `checks.module-eval-agent-icon` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + agentWith + runGroup + ; + + # An agent with an icon and no forge. Both halves of the avatar sync — the + # `.path` watcher and the oneshot it triggers — hang off the icon, but only + # the service can do anything with a forge URL, so the icon alone is the one + # input that can render half the feature. `pkgs.emptyFile` rather than a real + # SVG: the icon is only ever a gate here, and nothing this case reads + # rasterizes it. + agentIconNoForge = agentWith { services.hyperhive.agent.icon = pkgs.emptyFile; }; + cases = [ + { + # The absence class, and a gate asymmetry nothing else can see. A `.path` + # unit is `wantedBy = multi-user.target` and names a `Unit=` by + # convention rather than by reference, so one gated more loosely than the + # service it triggers evaluates clean, deploys clean, and then fails to + # activate the first time the watched file is written. `forge.url`'s own + # option doc already promises the avatar-sync units are "not generated at + # all" without a forge — this is the case that makes that sentence true + # of the watcher and not only of the oneshot. + name = "an agent with an icon and no forge renders neither avatar-sync unit"; + ok = + !(agentIconNoForge.systemd.paths ? forge-avatar-sync) + && !(agentIconNoForge.systemd.services ? forge-avatar-sync); + } + ]; +in +runGroup "agent-icon" cases diff --git a/nix/module-eval/agent-matrix.nix b/nix/module-eval/agent-matrix.nix new file mode 100644 index 00000000..2d25857d --- /dev/null +++ b/nix/module-eval/agent-matrix.nix @@ -0,0 +1,107 @@ +# `checks.module-eval-agent-matrix` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + runGroup + ; + + # Matrix's enable signal, which is the account set itself — there is no + # `matrix.enable` option left to read. Three arms, because the property has + # three distinct shapes and only one of them is the common case: + # + # - a homeserver URL, which is what the module turns into a `main` account; + # - neither URL nor operator account, the state that replaced + # `matrix.enable = false`. ⚠️ **This is the arm that matters.** `main` is + # declared by the module itself, so "any account declared" would be + # trivially true — and matrix would render for every agent in every hive — + # the moment that declaration stops being gated on the URL. Nothing else in + # this suite would notice; + # - an operator account carrying its own homeserver and no hive one, which is + # matrix on with no `main` at all. + agentMatrix = agent { matrix.url = "https://chat.t.local"; }; + + agentNoMatrix = agent { }; + + agentMatrixExternalOnly = agent { + matrixAccounts.ccc = { + tokenFile = "/agents/a1/state/matrix-token-ccc"; + sessionDir = "/agents/a1/state/matrix-sdk-state-ccc"; + homeserver = "https://matrix.example.invalid"; + }; + }; + cases = [ + { + # A homeserver URL is the whole input: from it the module derives the + # hive-internal `main` account, and from a non-empty account set the three + # things that used to hang off `matrix.enable`. + name = "an agent with a homeserver gets a main account and the matrix units"; + ok = + let + a = agentMatrix.services.hyperhive.agent.matrixAccounts; + in + lib.attrNames a == [ "main" ] + && a.main.tokenFile == "/agents/a1/state/matrix-token" + && a.main.homeserver == "https://chat.t.local" + && agentMatrix.systemd.services ? hive-matrix-daemon + && agentMatrix.systemd.paths ? hive-matrix-daemon + && agentMatrix.services.hyperhive.agent.extraMcpServers ? matrix; + } + { + # The absence arm, and the reason the enable signal is not vacuous. An + # agent the hive gave no homeserver, whose operator declared nothing, must + # come out with an EMPTY account set — not a `main` that can never log in + # — and therefore with none of the three. Assert the emptiness itself and + # not just the units: it is the account set that is load-bearing now, and + # a `main` sneaking back in is the regression this case exists to name. + name = "an agent with no homeserver and no declared account gets no matrix at all"; + ok = + agentNoMatrix.services.hyperhive.agent.matrixAccounts == { } + && !(agentNoMatrix.systemd.services ? hive-matrix-daemon) + && !(agentNoMatrix.systemd.paths ? hive-matrix-daemon) + && !(agentNoMatrix.services.hyperhive.agent.extraMcpServers ? matrix); + } + { + # Matrix without a hive homeserver: one operator account, its own + # homeserver, no `main`. Under the deleted `matrix.enable` this config was + # an assertion failure ("extras require enable") even though every account + # in it was complete; the account set being the signal is what makes it + # expressible, and the serialized env var is where that has to show up. + name = "an external-only account enables matrix with no main entry"; + ok = + let + accts = agentMatrixExternalOnly.services.hyperhive.agent.matrixAccounts; + env = agentMatrixExternalOnly.systemd.services.hive-matrix-daemon.environment; + in + lib.attrNames accts == [ "ccc" ] + && !(accts ? main) + && + builtins.fromJSON env.HIVE_MATRIX_ACCOUNTS == [ + { + name = "ccc"; + token_file = "/agents/a1/state/matrix-token-ccc"; + state_dir = "/agents/a1/state/matrix-sdk-state-ccc"; + homeserver = "https://matrix.example.invalid"; + } + ] + # No hive homeserver, so nothing may claim one. + && !(env ? HIVE_MATRIX_URL); + } + ]; +in +runGroup "agent-matrix" cases diff --git a/nix/module-eval/agent-memory.nix b/nix/module-eval/agent-memory.nix new file mode 100644 index 00000000..e1377e9c --- /dev/null +++ b/nix/module-eval/agent-memory.nix @@ -0,0 +1,87 @@ +# `checks.module-eval-agent-memory` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + runGroup + agentHarness + ; + + # The memory-pressure pair. `claudeMemoryMaxBytes` is the container's own + # cap, rendered per agent by meta.rs — the capped arm is the one every + # real deploy gets, the uncapped arm is a hive that set `infinity` or a + # RAM percentage and so hands the module no byte count to size against. + agentCapped = agent { claudeMemoryMaxBytes = 8589934592; }; + + agentUncapped = agent { }; + + agentSubagentDaemon = machine: machine.systemd.services.hive-subagent-daemon; + cases = [ + { + # Two thirds of the container's cap, and a SOFT ceiling: the daemon's + # cgroup holds every nested claude, so `MemoryHigh=` throttles the + # subagent set as a whole before the kernel picks a victim, while the + # absent `MemoryMax=` is what still lets one subagent use more than + # its share on a container that has the memory free. A hard cap here + # would trade the silent kill for a guaranteed wall, which is the + # shape this deliberately does not have. + name = "the subagent daemon throttles at two thirds of the container's memory"; + ok = + let + c = (agentSubagentDaemon agentCapped).serviceConfig; + in + c.MemoryHigh == "5726623061" && !(c ? MemoryMax); + } + { + # What makes the case above able to fail. With no byte count for the + # container there is no fraction to take, and a hardcoded fallback + # would be a number about some other hive's machine — so the unit + # renders no ceiling at all rather than a fabricated one. + name = "an agent with no byte-valued memory cap renders no subagent ceiling"; + ok = !((agentSubagentDaemon agentUncapped).serviceConfig ? MemoryHigh); + } + { + # The sign is the whole property, and it is easy to write backwards: + # a HIGHER OOMScoreAdjust is a MORE likely victim. So the subagent + # daemon must be strictly above zero and the harness strictly below + # it — swap the two and the kernel takes the agent's own turn first, + # which is worse than setting nothing at all. Both nested `claude` + # processes inherit their unit's value, so ordering the units orders + # the sessions. Held as an inequality rather than two constants: what + # must not drift is the order, not the magnitudes. + name = "the OOM killer prefers a subagent over the agent's own session"; + ok = + let + sub = (agentSubagentDaemon agentUncapped).serviceConfig.OOMScoreAdjust; + own = (agentHarness agentUncapped).serviceConfig.OOMScoreAdjust; + in + sub > 0 && own < 0 && sub > own; + } + { + # The pair the case above only makes sense with: preferring this unit + # as the victim is an improvement only if losing one subagent isn't + # losing all of them. systemd's default `stop` would take the daemon + # and every sibling session down with whichever process the kernel + # chose, which is the blast radius that made the preference a bad + # trade in the first place. + name = "one subagent losing the OOM draw does not stop the daemon"; + ok = (agentSubagentDaemon agentUncapped).serviceConfig.OOMPolicy == "continue"; + } + ]; +in +runGroup "agent-memory" cases diff --git a/nix/module-eval/agent-otel.nix b/nix/module-eval/agent-otel.nix new file mode 100644 index 00000000..a843362f --- /dev/null +++ b/nix/module-eval/agent-otel.nix @@ -0,0 +1,95 @@ +# `checks.module-eval-agent-otel` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + runGroup + ; + + # The log path's three hops, one fixture each. Nothing carries a journal + # record end to end at eval time, so what these defend is the part no tier + # can check for itself: each hop's output is the next hop's input, and + # every mismatch between them is silent — a push accepted and routed + # nowhere, a receiver pointed at an empty directory, a pipeline that does + # not exist. + agentBridge = "http://10.42.0.1:4318"; + + agentOtel = agent { + otel.enable = true; + otel.endpoint = agentBridge; + }; + + # The same agent over the other wire protocol. An exporter's NAME is what + # selects it, so this is where a defined exporter and the pipeline's + # reference to it can drift apart. + agentOtelGrpc = agent { + otel.enable = true; + otel.endpoint = agentBridge; + otel.protocol = "grpc"; + }; + + agentNoOtel = agent { }; + + agentSettings = machine: machine.services.opentelemetry-collector.settings; + cases = [ + { + # The journald receiver's own default directory is the RUNTIME + # journal, and a container that stores persistently leaves that + # empty. At the default the forwarder validates, starts, reports + # healthy and ships nothing, so this one literal is the difference + # between the path working and silently not. + name = "the agent forwarder reads the persistent journal, not the runtime one"; + ok = ((agentSettings agentOtel).receivers.journald.directory or null) == "/var/log/journal"; + } + { + # The hop's two ends: what it reads, and where what it reads goes. + # The endpoint is compared against the value the fixture handed the + # option rather than a literal spelled here, so an exporter that + # stopped reading the option fails instead of matching a constant + # that travelled beside it. + name = "the agent forwarder ships the journal to the endpoint its hive gave it"; + ok = + let + s = agentSettings agentOtel; + p = s.service.pipelines.logs; + in + p.receivers == [ "journald" ] + && p.exporters != [ ] + && lib.all (e: (s.exporters ? ${e}) && s.exporters.${e}.endpoint == agentBridge) p.exporters; + } + { + # Presence control for the two cases above: with the switch off there + # is no collector in the container at all, so their passing is about + # the wiring rather than about a unit that renders regardless. + name = "an agent that has not opted into telemetry runs no collector"; + ok = !agentNoOtel.services.opentelemetry-collector.enable; + } + { + # `otlp` and `otlphttp` are different components and the protocol + # option picks which one is defined. A pipeline left naming the other + # is a startup failure; an exporter no pipeline names is silence. + name = "the agent forwarder's exporter and its pipeline agree on the protocol"; + ok = + let + s = agentSettings agentOtelGrpc; + in + (s.exporters ? otlp) && s.service.pipelines.logs.exporters == [ "otlp" ]; + } + ]; +in +runGroup "agent-otel" cases diff --git a/nix/module-eval/agent-plugins.nix b/nix/module-eval/agent-plugins.nix new file mode 100644 index 00000000..f71e13b4 --- /dev/null +++ b/nix/module-eval/agent-plugins.nix @@ -0,0 +1,89 @@ +# `checks.module-eval-agent-plugins` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + runGroup + ; + + # `claudePlugins`'s additive-merge shape: a plain per-agent + # definition must ADD to the module's own base-set definition rather than + # replacing it, while `lib.mkForce` must still replace the whole list + # outright — the two arms below plus the unset default (read directly off + # `bare`-shaped `agent { }`, no fixture of its own needed) are the three + # cases that shape has to hold. + agentPluginsDefault = agent { }; + + agentPluginsAdded = agent { claudePlugins = [ "foo@bar" ]; }; + + agentPluginsForced = agent { claudePlugins = lib.mkForce [ "foo@bar" ]; }; + + # The de-dup arm: an agent that names a base-set entry explicitly must not + # get it installed twice. + agentPluginsDuplicate = agent { claudePlugins = [ "base@hyperhive" ]; }; + + agentPlugins = machine: machine.services.hyperhive.agent.claudePlugins; + cases = [ + { + # Case 1 of the additive-merge shape: an agent that declares + # nothing gets exactly the module's base set, no more and no less. + name = "an agent with no claudePlugins definition gets exactly the base set"; + ok = + agentPlugins agentPluginsDefault == [ + "skill-creator@claude-plugins-official" + "base@hyperhive" + ]; + } + { + # Case 2: a plain per-agent definition ADDS to the base set (list-typed + # options at equal priority concatenate) rather than replacing it — + # the property the operator ruling asked for instead of `mkDefault`. + # Sorted before comparing: concatenation order between two same- + # priority definitions is a module-system implementation detail this + # case isn't about — membership and count are. + name = "an agent's own claudePlugins definition adds to the base set"; + ok = + builtins.sort builtins.lessThan (agentPlugins agentPluginsAdded) == [ + "base@hyperhive" + "foo@bar" + "skill-creator@claude-plugins-official" + ]; + } + { + # Case 3: `lib.mkForce` is still the escape hatch — an operator who + # wants the base set gone, not extended, can still say so outright. + name = "an agent's mkForce claudePlugins replaces the base set outright"; + ok = agentPlugins agentPluginsForced == [ "foo@bar" ]; + } + { + # De-dup arm: an agent that names a base-set entry itself must not get + # it installed twice — `lib.unique` at the JSON-render site, not the + # option's merge (the merge is a plain concatenation on purpose, so + # the un-deduped list stays readable for `agentPlugins` above). + name = "an agent repeating a base-set plugin does not get it installed twice"; + ok = + builtins.sort builtins.lessThan ( + builtins.fromJSON agentPluginsDuplicate.environment.etc."hyperhive/claude-plugins.json".text + ) == [ + "base@hyperhive" + "skill-creator@claude-plugins-official" + ]; + } + ]; +in +runGroup "agent-plugins" cases diff --git a/nix/module-eval/agent-queue-bao.nix b/nix/module-eval/agent-queue-bao.nix new file mode 100644 index 00000000..6ebd09dd --- /dev/null +++ b/nix/module-eval/agent-queue-bao.nix @@ -0,0 +1,169 @@ +# `checks.module-eval-agent-queue-bao` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + agent + agentWith + runGroup + agentHarness + ; + + # The agent side of the swarm queue. Both coordinates set is the only state + # in which the harness unit declares a credential at all, so the pair and + # the empty fixture beside it are the two arms worth having. + agentQueue = agent { + queue.natsUrl = "nats://10.42.0.1:4222"; + queue.tokenEndpoint = "https://auth.t.local/api/oidc/token"; + }; + + agentNoQueue = agent { }; + + # The agent side of the swarm secret store. The address is the whole switch — + # it is both what generates the login check and what that check points at — + # so it and the empty fixture beside it are the two arms worth having. + agentBao = agentWith { services.hyperhive.agent.bao.addr = "https://bao.t.local:8200"; }; + + agentNoBao = agentWith { }; + + agentBaoIdentity = machine: machine.systemd.services.hive-agent-bao-identity; + cases = [ + { + # Both ids or neither: the secret authenticates nobody without the id it + # belongs to, and the harness refuses to treat one of the two as a queue. + name = "an agent with queue coordinates imports both halves of its credential"; + ok = + let + c = (agentHarness agentQueue).serviceConfig.LoadCredential; + in + builtins.elem "hive-queue-agent-secret" c && builtins.elem "hive-queue-agent-client-id" c; + } + { + # `%d` and not a path under the state dir: the host file is `0600` + # root-owned, so the only copy this unprivileged unit can open is the + # one systemd puts in its own credentials directory. + name = "the harness reads its queue credential out of the credentials directory"; + ok = + let + e = (agentHarness agentQueue).environment; + in + e.HIVE_AGENT_OIDC_CLIENT_SECRET_FILE == "%d/hive-queue-agent-secret" + && e.HIVE_AGENT_OIDC_CLIENT_ID_FILE == "%d/hive-queue-agent-client-id"; + } + { + # An agent built before its hive was handed the queue's address. It + # must declare nothing rather than name a credential that never + # arrives — and the harness then reports "no queue coordinates" + # instead of a half-set environment. + name = "an agent with no queue coordinates declares no credential"; + ok = + let + u = agentHarness agentNoQueue; + in + !(u.serviceConfig ? LoadCredential) + && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_SECRET_FILE) + && !(u.environment ? HIVE_AGENT_OIDC_CLIENT_ID_FILE); + } + { + # The three ids `hive_c0re::lifecycle::agent_identity` forwards under. + # Neither end can discover the other's spelling, and a mismatch is a + # credential that is simply not there — which this unit then reports as + # a hive that delivered nothing. + name = "an agent with the store enabled imports every half of its identity"; + ok = + let + c = (agentBaoIdentity agentBao).serviceConfig.LoadCredential; + in + builtins.elem "hive-agent-bao-cert" c + && builtins.elem "hive-agent-bao-key" c + && builtins.elem "hive-agent-bao-server-ca" c; + } + { + # `%d` and not a path under the agent's state dir, for the reason the + # queue arm above gives: the host file is `0600` to the hive daemon, so + # the only copy this unprivileged unit can open is the one systemd puts + # in its own credentials directory. The address is the option's value + # rather than a literal that agrees with it today. + name = "the identity check presents its certificate out of the credentials directory"; + ok = + let + e = (agentBaoIdentity agentBao).environment; + in + e.BAO_CLIENT_CERT == "%d/hive-agent-bao-cert" + && e.BAO_CLIENT_KEY == "%d/hive-agent-bao-key" + && e.BAO_ADDR == agentBao.services.hyperhive.agent.bao.addr; + } + { + # Same 403-not-a-miss reason as the hive-side readers: the path + # `swarm_secret_client::mtls::identity_path` builds is the one this + # agent's own policy stanza covers, and a path outside it is refused + # however correct it looks. Built from the agent's own name rather than + # from a literal, because the name is what makes it this agent's path + # and not some other agent's. + name = "the identity check reads the agent's own path"; + ok = + let + m = agentBao; + name = m.services.hyperhive.agent.user.name; + in + lib.hasInfix "secret/swarm/agents/${name}/bao-mtls" (agentBaoIdentity m).script; + } + { + # The whole point of the unit, and the thing a quieter default would + # undo: every arm of the check ends the unit non-zero, so an agent that + # cannot authenticate as itself says so at boot instead of at whichever + # pull needed the store first. + name = "the identity check fails the unit rather than degrading"; + ok = + let + u = agentBaoIdentity agentBao; + in + lib.hasInfix "exit 1" u.script + && !(lib.hasInfix "exit 0" u.script) + && u.serviceConfig.Restart == "on-failure"; + } + { + # Nothing about the identity may be printed, and the read-back is where + # that could slip: `bao kv get` on this path answers with certificate + # material, and the object beside it is a private key. The check needs + # only whether the read succeeded. + # + # The path goes through `lib.escapeShellArg` here for the same reason the + # module passes it through one — that helper decides whether an argument + # needs quotes at all, and this one (only `[a-z0-9/-]`) comes back bare. + # Spelling the quotes in by hand asserts a rendering nixpkgs chooses + # rather than the redirect this property is about. + name = "the identity check discards what it reads back"; + ok = + let + m = agentBao; + name = m.services.hyperhive.agent.user.name; + arg = lib.escapeShellArg "secret/swarm/agents/${name}/bao-mtls"; + in + lib.hasInfix "bao kv get -field=cert ${arg} >/dev/null" (agentBaoIdentity m).script; + } + { + # The absence arm, and what makes the four above able to fail. An agent + # whose swarm never minted an identity has nothing to log in with, and a + # failed unit at every boot would be the loudest possible statement + # about a deployment that never asked for one. + name = "an agent told no store address runs no identity check"; + ok = !(agentNoBao.systemd.services ? hive-agent-bao-identity); + } + ]; +in +runGroup "agent-queue-bao" cases diff --git a/nix/module-eval/bao-basics.nix b/nix/module-eval/bao-basics.nix new file mode 100644 index 00000000..ad39ff7f --- /dev/null +++ b/nix/module-eval/bao-basics.nix @@ -0,0 +1,219 @@ +# `checks.module-eval-bao-basics` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + baoNames + baoSettings + baoStream + bridgePorts + ; + + baoPkcs11 = hive { + deploy.bao.enable = true; + deploy.bao.seal = "pkcs11"; + }; + + baoShamir = hive { + deploy.bao.enable = true; + deploy.bao.seal = "shamir"; + }; + + baoExplicitCerts = hive { + deploy.bao.enable = true; + deploy.bao.serverCertFile = "/etc/pki/bao.pem"; + deploy.bao.serverKeyFile = "/etc/pki/bao-key.pem"; + }; + + # The store's units live inside its container, so the gates below have to + # look there rather than at the host's service set. + baoUnits = machine: machine.containers.swarm-bao.config.systemd.services; + cases = [ + { + # The store's seal is spread over six gates — the stanza, the + # provisioning unit, two bind mounts, a device and an EnvironmentFile. + # Rendering only some of them is the dangerous state: a store that + # says hardware-backed and seals with a software key, which no + # assertion can catch because every value is individually valid. + name = "a shamir store renders no TPM provisioning unit"; + ok = !(baoUnits baoShamir ? swarm-bao-token); + } + { + # Presence control for the case above. Without it, a typo in the + # option name would satisfy the absence arm forever. The second half is + # the fix itself: the unit has to run where openbao's `DynamicUser` is + # allocated, and a host unit writing the same bytes has no name to hand + # them to. + name = "a pkcs11 store provisions the token in the container, not on the host"; + ok = (baoUnits baoPkcs11 ? swarm-bao-token) && !(baoPkcs11.systemd.services ? swarm-bao-token); + } + { + # `allowedDevices` renders `DeviceAllow=` and nothing else — nspawn + # mounts its own /dev and cannot create device nodes, so permission to + # use a device that was never bound in opens nothing. Neither half + # fails on its own, which is why they are asserted as a pair. + name = "a pkcs11 store gets the TPM device bound in, not merely allowed"; + ok = + let + c = baoPkcs11.containers.swarm-bao; + in + (c.bindMounts ? "/dev/tpmrm0") && builtins.any (d: d.node == "/dev/tpmrm0") c.allowedDevices; + } + { + # The stanza's label and the label the unit creates are two literals that + # have to name one object, and the mechanism is asserted with them + # because it is valid only for an RSA key: openbao takes AES-GCM or + # RSA-OAEP and a TPM 2.0 has neither GCM nor an opinion about which the + # seal asked for. Every wrong combination renders and deploys, and + # surfaces as a pkcs11 error at `operator init`. + name = "the seal asks for the RSA key the provisioning unit creates"; + ok = + let + p = (baoSettings baoPkcs11).seal.pkcs11; + in + (p.mechanism or "") == "CKM_RSA_PKCS_OAEP" + && lib.hasInfix "--algorithm=rsa2048 --key-label=${p.key_label or ""}" ( + (baoUnits baoPkcs11).swarm-bao-token.script or "" + ); + } + { + # `DynamicUser` implies `ProtectSystem=strict`, so the token directory is + # read-only to the seal however it is owned, and the group is the only + # handle on a uid allocated at start. Dropping either surfaces as a + # pkcs11 error deep in a library, naming neither the mount nor the user. + name = "the store's seal may write the token directory, and is in both its groups"; + ok = + let + sc = (baoUnits baoPkcs11).openbao.serviceConfig; + in + # `or [ ]` rather than a bare select: the interesting mutation is the + # key being gone, and a select would abort the whole run with a nix + # trace instead of failing this case by name. + builtins.elem "/var/lib/swarm-bao-token" (sc.ReadWritePaths or [ ]) + && builtins.elem "swarm-bao-token" (sc.SupplementaryGroups or [ ]) + && builtins.elem "swarm-bao-tpm" (sc.SupplementaryGroups or [ ]); + } + { + # The device node belongs to the HOST and is matched by NUMBER, while the + # unit that opens it lives in the container — so the two sides holding + # the same gid is the entire mechanism. Letting either side auto-allocate + # renders cleanly, deploys cleanly, and leaves a 0660 node the seal + # cannot open. Compared rather than each checked against a literal: the + # property is that they AGREE, not what they agree on. + name = "the TPM group has the same gid on the host and inside the container"; + ok = + let + host = baoPkcs11.users.groups.swarm-bao-tpm.gid or null; + inner = baoPkcs11.containers.swarm-bao.config.users.groups.swarm-bao-tpm.gid or null; + in + host != null && host == inner; + } + { + # Absence arm for the case above — a shamir store never opens a TPM, so + # it must not claim a device node's group. Without this, pinning the gid + # unconditionally would look identical. + name = "a shamir store claims no TPM device group"; + ok = !(baoShamir.users.groups ? swarm-bao-tpm); + } + { + # The store's mTLS identity is a separate trust domain from both CAs in + # this tree, because it must not come from an authority the store will + # itself distribute. What supplies it is the glue, which mints a CA of + # the store's own — so an enabled store has all three paths, and if this + # ever reads null again the store stops coming up on its own. + name = "a deployed store is given its own certificate, key and client CA"; + ok = + let + b = baoPkcs11.services.hyperhive.deploy.bao; + in + b.serverCertFile != null && b.serverKeyFile != null && b.clientCaFile != null; + } + { + # Everything the glue sets is `mkDefault`, and this is the case that + # says so: a deployment whose certificates come from somewhere the glue + # has never heard of must win. Also the presence control for the case + # above — a renamed option would read `null` on both and satisfy + # neither, but only this one names a value. + name = "an operator's own certificate path beats the glue's default"; + ok = baoExplicitCerts.services.hyperhive.deploy.bao.serverCertFile == "/etc/pki/bao.pem"; + } + { + # Absence arm. A store with nothing to serve renders no reader, so the + # unit is a function of the PAIRING rather than of the store — which is + # the property that makes it glue instead of a feature of either side. + name = "a store with no homeserver beside it renders no token reader"; + ok = !(baoPkcs11.systemd.services ? swarm-bao-matrix-token); + } + { + # The name a reader dials has to resolve where the store runs; a + # multi-host swarm resolves it upstream instead. + name = "the store's host answers for the store's name"; + ok = builtins.elem "bao.t.local" (baoNames baoPkcs11); + } + { + # What makes that name reachable from inside an agent container, and the + # single reason it is a stream server rather than a vhost: `ssl_preread` + # routes on the SNI without decrypting, so the handshake bao completes is + # still the client's own and the certificate it authenticates by arrives + # intact. Terminating here would hand the store one identity for the + # whole swarm. + name = "the store's host passes connections through without terminating TLS"; + ok = + let + s = baoStream baoPkcs11; + in + lib.hasInfix "ssl_preread on;" s && lib.hasInfix "proxy_pass $swarm_bao_backend;" s; + } + { + # The address half, and it is bridge-only for a reason a wildcard would + # hide until deploy: the store already holds `127.0.0.1:8200` in this + # same netns, so `0.0.0.0:8200` is `EADDRINUSE` and nginx fails to start + # — taking every hive domain behind the gateway down with it. + name = "the passthrough listens on the bridge, not on every address"; + ok = lib.hasInfix "listen 10.42.0.1:8200;" (baoStream baoPkcs11); + } + { + # The routing half: the SNI picks the backend and the only name that + # resolves to one is the store's own. A `default` that pointed anywhere + # would make this host a relay for whatever name a client invented. + name = "the passthrough routes only the store's name, to its loopback listener"; + ok = + let + s = baoStream baoPkcs11; + in + lib.hasInfix "map $ssl_preread_server_name $swarm_bao_backend" s + && lib.hasInfix "bao.t.local 127.0.0.1:8200;" s + && lib.hasInfix ''default "";'' s; + } + { + # The listener is only half of reachable: the bridge firewall drops + # everything not named here, and a silent drop is the failure that reads + # as "the store is down" from inside a container. + name = "the store's port is open on the bridge where the store runs"; + ok = builtins.elem 8200 (bridgePorts baoPkcs11); + } + { + # Raft refuses to start without it, and says so in a message that names + # neither the setting nor the stanza. + name = "the store advertises a cluster address"; + ok = lib.hasPrefix "https://" ((baoSettings baoPkcs11).cluster_addr or ""); + } + ]; +in +runGroup "bao-basics" cases diff --git a/nix/module-eval/bao-controller.nix b/nix/module-eval/bao-controller.nix new file mode 100644 index 00000000..22e1e573 --- /dev/null +++ b/nix/module-eval/bao-controller.nix @@ -0,0 +1,216 @@ +# `checks.module-eval-bao-controller` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # Store and controller on one machine, with a CN no default could supply. + # The odd value is what lets the case below tell "both ends read the same + # option" from "both ends happen to say swarm-controller". + baoControllerHere = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.controllerCommonName = "cn-marker-not-a-default"; + deploy.swarm-controller.enable = true; + }; + + # The controller with no store, which is every spread deployment. Nothing + # mints here, so the pairing must leave the paths unset rather than name + # files this host will never have. + controllerNoStore = hive { deploy.swarm-controller.enable = true; }; + + # The two authorities told apart. A deployment that self-signs both ends + # points `clientCaFile` and `serverCaFile` at one file, so on the fixture + # above the CA a hive is issued from and the CA the store is verified by are + # the same string — and a case wiring either into the other's slot passes. + # This is the deployment where they differ, which is what makes the arm + # below able to fail at all. + controllerTwoCas = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.swarm-controller.enable = true; + deploy.bao.clientCaFile = lib.mkForce "/etc/pki/hive-clients-ca.pem"; + deploy.bao.serverCaFile = lib.mkForce "/etc/pki/store-server-ca.pem"; + }; + + # The host's `bao` wrapper, pulled apart once so each case below names one + # property instead of a conjunction — a failing conjunction says only that + # something is wrong. + baoHostPackages = controllerTwoCas.environment.systemPackages; + + baoWrapper = lib.findFirst (p: (p.name or "") == "bao-hive") null baoHostPackages; + + baoWrapperCmd = if baoWrapper == null then "" else (baoWrapper.buildCommand or ""); + cases = [ + { + # Nothing asserted the PKI script before this, so a third leaf could be + # added to it and every case still passed — measured, not assumed: the + # commit that added one left `module-eval`'s derivation unchanged. + name = "the store mints a leaf for the controller, and the controller is pointed at it"; + ok = + let + m = baoControllerHere; + pki = m.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "controller.pem" pki + && + m.services.hyperhive.deploy.swarm-controller.baoClientCertFile + == "/var/lib/swarm-bao-pki/controller.pem" + && + m.services.hyperhive.deploy.swarm-controller.baoClientKeyFile + == "/var/lib/swarm-bao-pki/controller-key.pem"; + } + { + # What makes the one above mean something: a controller with no store + # has nothing to be pointed at. Naming a path here would be a file this + # host never gets, which fails at a TLS handshake rather than at eval. + name = "a controller on a host with no store is left without certificate paths"; + ok = + let + c = controllerNoStore.services.hyperhive.deploy.swarm-controller; + in + c.baoClientCertFile == null && c.baoClientKeyFile == null; + } + { + # Being *pointed at* a leaf and *being handed* one are different claims, + # and the options above were the first without the second — declared, + # defaulted, and read by nothing. This is the arm that makes them reach + # the process. + # + # ⚠️ The LoadCredential source is asserted, not just the `%d` name: the + # controller's leaf and the hive reader's are two identities with two + # policies, and wiring `deploy.bao.clientCertFile` here would satisfy + # every `%d`-only check while giving the daemon a policy that cannot + # write an agent's credential. + name = "the controller is handed its own store leaf, not the hive reader's"; + ok = + let + s = baoControllerHere.systemd.services; + in + s ? swarm-controller + && (s.swarm-controller.environment ? BAO_ADDR) + && (s.swarm-controller.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" + && (s.swarm-controller.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" + && builtins.elem "bao-client.pem:/var/lib/swarm-bao-pki/controller.pem" s.swarm-controller.serviceConfig.LoadCredential + && builtins.elem "bao-client-key.pem:/var/lib/swarm-bao-pki/controller-key.pem" s.swarm-controller.serviceConfig.LoadCredential; + } + { + # A hive's cert-auth role carries the authority by value, so the daemon + # has to be handed the file rather than a path into the store's own + # directory it cannot read. + # + # ⚠️ The LoadCredential source is asserted, not just the `%d` name, for + # the reason the arm above gives — and here the wrong file is a + # *plausible* one: `deploy.bao.serverCaFile` is the CA a reader checks + # the store's certificate with, evaluates fine in this slot, and would + # make every hive role trust the wrong authority. + name = "the controller is handed the CA hives are issued from"; + ok = + let + s = controllerTwoCas.systemd.services; + m = controllerTwoCas.services.hyperhive; + in + (s.swarm-controller.environment.SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE or null) + == "%d/hive-client-ca.pem" + && builtins.elem "hive-client-ca.pem:/etc/pki/hive-clients-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && !builtins.elem "hive-client-ca.pem:/etc/pki/store-server-ca.pem" s.swarm-controller.serviceConfig.LoadCredential + && m.deploy.swarm-controller.hiveClientCaFile == m.deploy.bao.clientCaFile; + } + { + # Same two-CA fixture, for the same reason: the wrapper verifies the + # STORE, so it takes `serverCaFile`. On a self-signing deployment both + # options name one file and either would pass; here the client CA in that + # slot is a case this arm fails. + # + # ⚠️ The package itself stays off `PATH` — `wrapProgram` renames the real + # binary, so an unwrapped `bao` is unreachable rather than merely + # discouraged. Operator's instruction, and the last assertion is what + # keeps a later "install the package too" from quietly undoing it. + name = "the host gets a wrapped bao CLI"; + ok = baoWrapper != null; + } + { + name = "the wrapped bao CLI carries this store's address"; + ok = lib.hasInfix "--set-default BAO_ADDR" baoWrapperCmd; + } + { + # `serverCaFile` and not `clientCaFile`: the wrapper verifies the STORE. + # On a self-signing deployment both options name one file and either + # would pass, which is why this uses the two-CA fixture. + # + # ⚠️ The flag and its VALUE together, escaped the same way the module + # escapes it: `BAO_CACERT` present and `store-server-ca.pem` present + # somewhere are two facts that do not add up to "the CA is set to that + # file", and a weaker pair of `hasInfix`es passes on a wrapper that sets + # neither to the other. + name = "the wrapped bao CLI verifies the store with the server CA"; + ok = + lib.hasInfix "--set-default BAO_CACERT ${lib.escapeShellArg "/etc/pki/store-server-ca.pem"}" baoWrapperCmd + && !lib.hasInfix "hive-clients-ca.pem" baoWrapperCmd; + } + { + # `wrapProgram` renames the real binary, so an unwrapped `bao` is + # unreachable rather than merely discouraged — operator's instruction. + # This is what keeps a later "install the package too" from undoing it. + name = "the unwrapped bao package stays off the host PATH"; + ok = !builtins.elem controllerTwoCas.services.hyperhive.deploy.bao.package baoHostPackages; + } + { + # Absence arm for the one above: without a store identity there is + # nothing to write a role with, so handing over the authority would be + # giving a file to a daemon that cannot act on it. + name = "a controller with no store leaf is given no hive CA either"; + ok = + let + s = controllerNoStore.systemd.services; + in + !(s.swarm-controller.environment ? SWARM_CONTROLLER_HIVE_CLIENT_CA_FILE) + && !(lib.any (c: lib.hasPrefix "hive-client-ca" c) s.swarm-controller.serviceConfig.LoadCredential); + } + { + # Absence arm for the one above, and what makes it mean anything: a + # controller with no leaf gets no store environment at all rather than + # variables naming files this host never receives. + name = "a controller with no store leaf is given no store environment"; + ok = + let + s = controllerNoStore.systemd.services; + in + s ? swarm-controller + && !(s.swarm-controller.environment ? BAO_ADDR) + && !(lib.any (c: lib.hasPrefix "bao-" c) s.swarm-controller.serviceConfig.LoadCredential); + } + { + # The CN is an interface between two files: the store writes a role that + # matches it, the PKI mints a leaf that carries it. They read one option, + # and this is what says so — the fixture's value cannot come from a + # default, so matching it in both places is not a coincidence. + name = "the cert-auth role and the minted leaf take their subject from one option"; + ok = + let + m = baoControllerHere; + role = m.systemd.services.swarm-bao-controller-policy.script; + pki = m.systemd.services.swarm-bao-pki.script; + in + lib.hasInfix "cn-marker-not-a-default" role && lib.hasInfix "cn-marker-not-a-default" pki; + } + ]; +in +runGroup "bao-controller" cases diff --git a/nix/module-eval/bao-grants.nix b/nix/module-eval/bao-grants.nix new file mode 100644 index 00000000..256b1cee --- /dev/null +++ b/nix/module-eval/bao-grants.nix @@ -0,0 +1,236 @@ +# `checks.module-eval-bao-grants` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # The store, plus a placed bootstrap token: the only shape in which the + # swarm's first grant can be written at all. + baoGrantHere = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + + # The credential without the store. Writing the first grant is a store-side + # operation, so a host holding only the token has nothing to do — and this + # is the arm that separates "an operator placed a token" from "this box can + # act on it". + baoGrantNoStore = hive { + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + }; + + # The store and the token, with no CA to trust. `mkForce` because the PKI + # glue supplies one by default here — this is the deployment that brings its + # own certificates and has not named the authority yet, and it separates + # "the grant unit runs" from "cert auth can be set up". + baoGrantNoClientCa = hive { + deploy.bao.enable = true; + deploy.bao.bootstrapTokenFile = "/run/secrets/bao-bootstrap.token"; + deploy.bao.clientCaFile = lib.mkForce null; + }; + cases = [ + { + # Reads the rendered unit on the HOST, which is where the write happens: + # every API listener demands a client certificate, and the host is the + # side that has one. + name = "a store host with a placed bootstrap token renders the granting unit on the host"; + ok = + let + u = baoGrantHere.systemd.services.swarm-bao-controller-policy; + in + lib.hasInfix "/run/secrets/bao-bootstrap.token" u.script + && u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token"; + } + { + # The move is the fix, so pin the side it landed on: in the container it + # had no identity to open a connection with, and no address that resolved + # to the store from its own netns. + name = "the granting unit is not rendered inside the store's container"; + ok = !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-controller-policy); + } + { + # `StartLimit*` are `[Unit]` settings that systemd ignores under + # `[Service]`, so a bound written into `serviceConfig` renders, deploys + # and does nothing. Asserted where nixpkgs puts it rather than where it + # was written. The values are pinned because they are the bound: under + # `shamir` a human unseals by hand, and anything shorter than a day gives + # up first — `start-limit-hit` does not self-heal. + name = "the granting unit's start limit lands in [Unit], not [Service]"; + ok = + let + u = baoGrantHere.systemd.services.swarm-bao-controller-policy; + in + toString u.unitConfig.StartLimitBurst == "2880" + && toString u.unitConfig.StartLimitIntervalSec == "90000" + && !(u.serviceConfig ? StartLimitBurst); + } + { + # The grants themselves, and the `hive-` prefix is the whole point: + # without it the controller can rewrite the policy that constrains it, + # which is a privilege escalation that renders, deploys and looks fine. + # Readable here only because the HCL is piped as an argument rather than + # written to a store path. + name = "the controller's bao grants cannot reach the policy that constrains it"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "sys/policies/acl/hive-*" s && !(lib.hasInfix "sys/policies/acl/*" s); + } + { + # Same host-side reasoning as the controller's granting unit above: the + # write needs a client certificate and the host is the side that has one. + name = "a store host with a placed bootstrap token renders the publisher's granting unit too"; + ok = + let + u = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy; + in + u.unitConfig.ConditionPathExists == "/run/secrets/bao-bootstrap.token" + && lib.hasInfix "swarm-secret-publisher" u.script; + } + { + # The control for the case above, and the same one the controller's unit + # has: rendered on the host means NOT rendered in the container, where it + # would have neither an identity nor a route to the store. + name = "the publisher's granting unit is not rendered inside the store's container"; + ok = + !(baoGrantHere.containers.swarm-bao.config.systemd.services ? swarm-bao-secret-publisher-policy); + } + { + # The whole point of a second principal. The two prefixes it publishes to + # and not `swarm/`, so it cannot touch an agent's credentials; and no + # `read`, so a unit whose job is copying a file cannot recover what is + # already there. Pinned as the full capability list per prefix, because an + # added capability is exactly what a presence check misses. + name = "the publisher's grant is write-only and reaches the hive and service prefixes alone"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/hives/*\" {\n capabilities = [\"create\", \"update\"]" s + && lib.hasInfix "path \"secret/data/swarm/services/*\" {\n capabilities = [\"create\", \"update\"]" s + && !(lib.hasInfix "secret/data/swarm/agents" s) + && !(lib.hasInfix "secret/data/swarm/*" s) + && !(lib.hasInfix "sys/policies/acl" s); + } + { + # The ordering is load-bearing and invisible at runtime: the controller's + # unit creates the KV and cert-auth mounts this one writes into, so + # without it a cold boot races and fails with "route entry not found", + # which names neither unit. + name = "the publisher's granting unit is ordered after the one that creates the mounts"; + ok = lib.elem "swarm-bao-controller-policy.service" ( + baoGrantHere.systemd.services.swarm-bao-secret-publisher-policy.after + ); + } + { + # The policy authorising this route lives in another file, and nothing + # else relates the grants to the paths the code actually writes. + # + # `secret/data/` is KV v2's ACL prefix; `swarm` is + # `swarm_secret_client::path::ROOT` and `agents` is + # `Kind::Agent.as_str()`, both of which that crate pins in its own test. + # + # The grant is still the agent kind alone because nothing writes another + # one yet. It widens when a path outside `agents/` gains a writer, not + # when the kinds are declared. + name = "the controller may write agent credentials, and only under the agent prefix"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "secret/data/swarm/agents/*" s + && !(lib.hasInfix "secret/data/*" s) + && !(lib.hasInfix "path \"secret/*\"" s); + } + { + # Write-only is the property, not an accident of how it was typed: a + # `read` here would let the controller recover every agent's credentials + # instead of only replacing them. Pinned as the whole capability list, + # because an added capability is exactly what a presence check misses. + name = "the controller's grant on agent credentials is write-only"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "path \"secret/data/swarm/agents/*\" {\n capabilities = [\"create\", \"update\"]" s; + } + { + # The policy above grants paths under a mount nothing else creates, so + # the unit that writes the policy has to create it too — otherwise every + # certificate login fails against a path that is not there. + name = "the granting unit creates the cert auth mount and the controller's role"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "bao auth enable cert" s + && lib.hasInfix "auth/cert/certs/swarm-controller" s + && lib.hasInfix "/var/lib/swarm-bao-tls/client-ca.pem" s; + } + { + # Same shape as the cert mount above, for the engine the controller + # writes credentials through: a fresh store has no `secret/`, so the + # grant would name a mount nobody created and the first write would 404. + # + # ⚠️ Matched on the COMMAND, for the reason the no-client-CA case below + # spells out: the policy text is embedded in this same script and grants + # `secret/data/...`, so any arm keyed on the *path* is satisfied either + # way and could never fail. + name = "the granting unit creates the KV mount the controller writes through"; + ok = + let + s = baoGrantHere.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "bao secrets enable -path=secret kv-v2" s; + } + { + # The arm that makes the one above mean something. A role's trust anchor + # is the CA, so with none named there is nothing to write — and the + # policy write, which needs no CA, must survive that. + # + # ⚠️ Matched on the COMMANDS, not on `auth/cert/certs`: the policy text is + # embedded in this same script and grants that very path, so the shorter + # infix is present either way and the arm could never fail. + name = "with no client CA the unit still writes the policy and skips the role"; + ok = + let + s = baoGrantNoClientCa.systemd.services.swarm-bao-controller-policy.script; + in + lib.hasInfix "bao policy write" s + && !(lib.hasInfix "bao auth enable cert" s) + && !(lib.hasInfix "client-ca.pem" s) + # The KV mount is NOT part of what a missing client CA switches off: + # the controller writes through it whether or not anything can log in + # by certificate. Asserted here rather than trusted, because both + # steps live in the same script and one indentation level decides it. + && lib.hasInfix "bao secrets enable -path=secret kv-v2" s; + } + { + # What makes the granting-unit cases mean something, and the property + # the host-side half depends on: no store here, so no bind mount and no + # unit. Without it a hive that merely names a token would drag the + # store's container config into its evaluation. + name = "a bootstrap token on a host that runs no store grants nothing"; + ok = !(baoGrantNoStore.systemd.services ? swarm-bao-bootstrap-dir); + } + ]; +in +runGroup "bao-grants" cases diff --git a/nix/module-eval/bao-matrix-reader.nix b/nix/module-eval/bao-matrix-reader.nix new file mode 100644 index 00000000..29cd5fdf --- /dev/null +++ b/nix/module-eval/bao-matrix-reader.nix @@ -0,0 +1,224 @@ +# `checks.module-eval-bao-matrix-reader` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # The store and a service that reads from it, versus the store alone. The + # pair is what makes the reader's absence arm mean anything. + baoWithMatrix = hive { + deploy.bao.enable = true; + deploy.matrix.enable = true; + }; + + # A hive that reads from a store it does not run: no `deploy.bao.enable`, so + # nothing here mints a leaf and the operator names one placed by hand. The + # deployment this pairing exists to serve, and the one that was previously + # inexpressible — the gate asked whether the store was a neighbour. + baoRemoteReader = hive { + deploy.matrix.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + cases = [ + { + # A login failure is the store being unreachable, sealed, or not yet + # holding this host's role — all of which a retry fixes. A read that + # answers "nothing there" is not, so only the first is allowed to fail + # the unit. + name = "the matrix token reader retries a failed login and still degrades on an empty read"; + ok = + let + u = baoWithMatrix.systemd.services.swarm-bao-matrix-token; + # Everything between the login's failure branch and the read's, which + # is where the exit that decides "retry or give up" lives. + afterLogin = lib.last (lib.splitString "bao login" u.script); + loginBranch = lib.head (lib.splitString "bao kv get" afterLogin); + in + u.serviceConfig.Restart or null == "on-failure" + && u.startLimitBurst or 0 > 0 + # The window has to outlast every attempt, or the burst is unreachable. + && u.startLimitIntervalSec or 0 > (u.serviceConfig.RestartSec or 0) * (u.startLimitBurst or 0) + && lib.hasInfix "exit 1" loginBranch + && lib.hasInfix "exit 0" (lib.last (lib.splitString "bao kv get" u.script)); + } + { + # The reader's own grant covers `swarm/hives//*` and + # `swarm/agents/*`; a path outside those answers 403, not "no such key". + # So the hive segment is what makes the read reachable, and a rename that + # drops it looks correct and fails identically on every boot. + name = "the matrix token path sits inside the prefix the reader is granted"; + ok = + let + s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; + in + lib.hasInfix "secret/swarm/hives/" s + && lib.hasInfix "/matrix/appservice-token" s + # The shape it used to have: `matrix` where a principal kind belongs, + # which no grant covers. + && !(lib.hasInfix "secret/swarm/matrix/" s); + } + { + # The store's second reader, and the gate that decides it exists is the + # certificate rather than anything about agents: containers are created + # at runtime, so there is no static "this hive runs agents" fact to ask. + name = "a hive that names a client identity reads its agent queue credential"; + ok = baoRemoteReader.systemd.services ? swarm-bao-queue-agent; + } + { + # Same 403-not-a-miss reason as the matrix arm above, against the path + # `swarm_secret_client::queue::agent_client_path` builds from the same + # pieces. The negative arm is the rename this one is exposed to: a + # credential named for the queue rather than for the hive that presents + # it reads as correct and is refused on every boot. + name = "the agent queue credential path sits inside the prefix the reader is granted"; + ok = + let + s = baoRemoteReader.systemd.services.swarm-bao-queue-agent.script; + in + lib.hasInfix "secret/swarm/hives/h1/queue/agent" s && !(lib.hasInfix "secret/swarm/queue/" s); + } + { + # The unit's output is the option's value, not a literal that agrees with + # it today: an operator moving the directory has to move both files. The + # prefix is asserted too because `hasInfix ""` is true — an option + # renamed out from under this arm would otherwise read empty and pass. + name = "the queue credential reader writes both files under the directory its option names"; + ok = + let + m = baoRemoteReader; + dir = toString m.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; + s = m.systemd.services.swarm-bao-queue-agent.script; + in + lib.hasPrefix "/var/lib/" dir + && lib.hasInfix "${dir}/secret" s + && lib.hasInfix "${dir}/client_id" s; + } + { + # A reader off the store's host is a reader whose journal is the only + # record of why a hive's agents never connected, so the collector has to + # be told the unit exists. Nothing else can say it: the store's module + # does not know who holds a certificate. + name = "the queue credential reader's journal reaches the collector"; + ok = builtins.elem "swarm-bao-queue-agent" baoRemoteReader.services.hyperhive.swarm.otel.journaldUnits; + } + { + # No agent container may render before this unit has had its attempts, + # and the edge that guarantees it must delay hive-c0re rather than sink + # it: an unreachable store is this unit's `Restart=on-failure` window, + # not a reason for the daemon that renders every agent to fail its own + # start. + name = "the queue credential reader orders before hive-c0re and is wanted, not required, by it"; + ok = + let + u = baoRemoteReader.systemd.services.swarm-bao-queue-agent; + in + builtins.elem "hive-c0re.service" (u.before or [ ]) + && builtins.elem "hive-c0re.service" (u.wantedBy or [ ]) + && !(builtins.elem "hive-c0re.service" (u.requiredBy or [ ])) + && !(builtins.elem "hive-c0re.service" (u.requires or [ ])); + } + { + # The store's first reader. Its unit belongs to the pairing, not to + # either service: matrix must not learn the store exists, and the store + # must not know who reads it. + name = "a store deployed beside the homeserver fetches its appservice token"; + ok = baoWithMatrix.systemd.services ? swarm-bao-matrix-token; + } + { + name = "a hive that names a client identity reads from a store it does not run"; + ok = baoRemoteReader.systemd.services ? swarm-bao-matrix-token; + } + { + # `Requires=` on a unit that does not exist fails the job, and nothing + # local mints certificates off-host — so this orders against nothing. + # Eval cannot see that failure; only the empty list here stands in for it. + name = "an off-host reader requires no unit the store's host would have provided"; + # Membership first, then the value: indexing a missing unit throws, and a + # table that reports which property broke must not be the thing that dies. + ok = + let + s = baoRemoteReader.systemd.services; + in + s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ ]; + } + { + # Presence control for the case above: the list is conditional, not gone. + name = "a co-located reader still orders after the local pki unit"; + ok = + let + s = baoWithMatrix.systemd.services; + in + s ? swarm-bao-matrix-token && s.swarm-bao-matrix-token.requires == [ "swarm-bao-pki.service" ]; + } + { + # Nothing asserted this script before, which is how it kept a branch that + # named three states and threw away the only thing telling them apart. A + # missing value, a refused identity and an unreachable host all end in the + # same degraded mode here, correctly — what must survive is which one. + name = "the matrix token reader carries the store's own diagnostic into the journal"; + ok = + let + s = baoWithMatrix.systemd.services.swarm-bao-matrix-token.script; + in + !(lib.hasInfix "2>/dev/null" s) && lib.hasInfix ''cat "''$err"'' s; + } + { + # hive-c0re runs as hive-core and the client key is `0600` root-owned + # inside a `0700` directory, so the identity reaches the daemon as a + # systemd credential and the environment names `%d` rather than the + # file. Both halves are asserted together because either alone is a + # daemon that fails at the TLS handshake, naming neither. + name = "a reader hands hive-c0re a store identity the daemon cannot open itself"; + ok = + let + s = baoRemoteReader.systemd.services; + in + s ? hive-c0re + && (s.hive-c0re.environment.BAO_CLIENT_CERT or null) == "%d/bao-client.pem" + && (s.hive-c0re.environment.BAO_CLIENT_KEY or null) == "%d/bao-client-key.pem" + && builtins.elem "bao-client.pem:/etc/pki/bao-client.pem" s.hive-c0re.serviceConfig.LoadCredential + && builtins.elem "bao-client-key.pem:/etc/pki/bao-client-key.pem" s.hive-c0re.serviceConfig.LoadCredential; + } + { + # The CA is its own arm: absent means the system trust store, which is + # right for a deployment with a real CA and wrong for a self-signed one. + name = "a reader that names no store CA falls through to the system trust store"; + ok = + let + s = baoRemoteReader.systemd.services; + in + s ? hive-c0re && !(s.hive-c0re.environment ? BAO_CACERT); + } + { + # Presence control for the arm above: the CA is conditional, not gone. + # Co-located, ./host-modules/glue-bao-tls.nix mints one and names it. + name = "a reader beside a self-signed store is given that store's CA"; + ok = + let + s = baoWithMatrix.systemd.services; + in + s ? hive-c0re + && (s.hive-c0re.environment.BAO_CACERT or null) == "%d/bao-ca.pem" + && lib.any (c: lib.hasPrefix "bao-ca.pem:" c) s.hive-c0re.serviceConfig.LoadCredential; + } + ]; +in +runGroup "bao-matrix-reader" cases diff --git a/nix/module-eval/bao-otel-collector.nix b/nix/module-eval/bao-otel-collector.nix new file mode 100644 index 00000000..4558d17b --- /dev/null +++ b/nix/module-eval/bao-otel-collector.nix @@ -0,0 +1,131 @@ +# `checks.module-eval-bao-otel-collector` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + baoSettings + ; + + baoTwoAddresses = hive { + deploy.bao.enable = true; + deploy.bao.extraListenAddresses = [ "10.0.0.1" ]; + # Pinned, not incidental: the case counting these listeners is about the + # declared addresses, and a collector on this host would add one of its own. + deploy.swarm-otel.enable = false; + }; + + # The store with and without a collector on the same host. `scrapeTargets` + # is only ever read by a local collector, so the metrics endpoint is a + # function of the pairing rather than of the store. + baoWithCollector = hive { + deploy.bao.enable = true; + deploy.swarm-otel.enable = true; + # A second job declared as bare `host:port`, so the pair of cases below + # reads one rendered scrape list: the store's entry carries a path, this + # one carries none. + swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; + }; + + baoNoCollector = hive { + deploy.bao.enable = true; + deploy.swarm-otel.enable = false; + }; + + # The scrape list prometheus is handed, not the option a service declared: + # the address, the path and the query are one string on the way in and three + # fields on the way out, and only the second shape is what gets requested. + scrapeJob = + machine: job: + lib.findFirst (c: c.job_name == job) null + machine.containers.swarm-otel.config.services.opentelemetry-collector.settings.receivers.prometheus.config.scrape_configs; + cases = [ + { + # Same gap one tier up, and it needs its own arm: this collector + # already had six scrape targets, so a pass here is about the seventh + # rather than about the receiver existing at all. + name = "the swarm collector scrapes its own telemetry endpoint"; + ok = + let + j = scrapeJob baoWithCollector "collector"; + in + j != null && j.static_configs == [ { targets = [ "127.0.0.1:8889" ]; } ]; + } + { + # The store stays behind the passthrough rather than beside it: loopback + # plus whatever was declared, never the bridge. A store that also bound + # the bridge itself would collide with the listener above, and the + # colliding one is nginx — the whole gateway, not just this port. + name = "the store binds loopback and its declared addresses, never the bridge"; + ok = + let + l = (baoSettings baoTwoAddresses).listener; + in + l.loopback.address == "127.0.0.1:8200" && l.extra-1.address == "10.0.0.1:8200"; + } + { + # Control for the case above: these settings are rendered per deployment, + # not constants a passing case could be indifferent to. + name = "a declared extra address renders a second listener beside loopback"; + ok = builtins.length (builtins.attrNames (baoSettings baoTwoAddresses).listener) == 2; + } + { + # Retention is what serves the endpoint at all, so the listener alone + # would be a port that answers 404. + name = "a store beside a collector serves metrics on its own listener"; + ok = + let + s = baoSettings baoWithCollector; + in + s.listener ? metrics && (s.telemetry.prometheus_retention_time or "0s") != "0s"; + } + { + # openbao serves no `/metrics` at all, so a scrape of the default path + # 404s: the store looks like a dead exporter, and every panel built on + # it renders empty rather than erroring. + name = "the store's scrape asks for the path openbao serves"; + ok = + let + j = scrapeJob baoWithCollector "bao"; + in + (j.metrics_path or "") == "/v1/sys/metrics" && (j.params.format or [ ]) == [ "prometheus" ]; + } + { + # Presence control for the case above: both fields are omitted rather + # than defaulted, so a target declared as bare `host:port` renders what + # it rendered before the path grammar existed. + name = "a target with no path renders neither metrics_path nor params"; + ok = + let + j = scrapeJob baoWithCollector "plain"; + in + j != null && !(j ? metrics_path) && !(j ? params); + } + { + # Absence arm. Unauthenticated by design, so it must not exist where + # nothing reads it. + name = "a store with no collector beside it serves no metrics"; + ok = + let + s = baoSettings baoNoCollector; + in + !(s.listener ? metrics) && !(s ? telemetry); + } + ]; +in +runGroup "bao-otel-collector" cases diff --git a/nix/module-eval/core-toggle.nix b/nix/module-eval/core-toggle.nix new file mode 100644 index 00000000..d7b587dc --- /dev/null +++ b/nix/module-eval/core-toggle.nix @@ -0,0 +1,422 @@ +# `checks.module-eval-core-toggle` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + baoNames + baoStream + bridgePorts + bridgePortsOrNone + swarmServiceEnables + ; + + allLocal = hive { deploy.singleHostSwarm = true; }; + + bare = hive { }; + + # The same stub with the central toggle off. Paired with `bare` below to pin + # the defaults that used to read `services.hyperhive.enable` and no longer + # do: each is asserted to hold the SAME literal in both, so a future edit + # that quietly re-introduces the dependency — or that changes what the + # default renders for a hive with the toggle on — fails here. Reading an + # option off this fixture forces that option only, not the config, so the + # toggle being off costs nothing. Also the "installs the modules and turns + # nothing on" host the swarm-service absences below read: none of the + # per-service deployment toggles derives from the hive being on, so it + # renders the same absences `bare` does. + centralToggleOff = hive { enable = false; }; + + withCi = hive { deploy.forgejo.ci.enable = true; }; + + # A priority collision is a property of the *option*, not + # of the merged value's interior — nix throws the moment the value is + # demanded at all, so `seq`-ing each `serviceConfig` value to WHNF is + # both necessary and sufficient. `deepSeq` over-specifies this: it keeps + # walking *into* the resulting value after the merge already succeeded, + # and a package/derivation-shaped value's `override`/`overrideAttrs` + # self-reference sends it into nixpkgs' fixpoint machinery and blows the + # stack (measured — this is not a hypothetical). + forceCiServiceConfigs = + let + svcs = withCi.containers.hive-ci.config.systemd.services; + vals = lib.concatMap (s: builtins.attrValues (s.serviceConfig or { })) (builtins.attrValues svcs); + in + builtins.foldl' (acc: v: builtins.seq v acc) true vals; + cases = [ + { + # Both halves matter. The equality is the "no longer consults the central + # toggle" half; the literal is the "and still renders what it always + # did" half, which an equality on its own would let drift to `false` in + # lockstep. + name = "the forge's behindGateway default is true regardless of the central toggle"; + ok = + bare.services.hyperhive.deploy.forgejo.behindGateway == true + && centralToggleOff.services.hyperhive.deploy.forgejo.behindGateway == true; + } + { + # Downstream of the one above — publicUrl reads `behindGateway`, so it + # tracked the central toggle transitively as well as directly. The domain + # is the stub's swarm domain, which both fixtures share. + name = "the forge's publicUrl default follows behindGateway alone, not the central toggle"; + ok = + bare.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local" + && centralToggleOff.services.hyperhive.swarm.forge.publicUrl == "https://forge.t.local"; + } + { + # And that it still tracks `behindGateway` at all: without this arm the + # case above passes just as well for a default hardcoded to the URL. + name = "the forge's publicUrl default is still null with behindGateway off"; + ok = + (hive { deploy.forgejo.behindGateway = false; }).services.hyperhive.swarm.forge.publicUrl == null; + } + { + # The controller's token path defaulted to forge's delivery path only on + # a host with the central toggle on, and to `null` otherwise. Forge + # deploys unconditionally, so the path is now unconditional too. + name = "the swarm controller's forgeTokenFile defaults to forge's delivery path regardless of the central toggle"; + ok = + let + forgePath = "/var/lib/hyperhive-forge/swarm-controller.token"; + in + bare.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath + && centralToggleOff.services.hyperhive.deploy.swarm-controller.forgeTokenFile == forgePath; + } + { + name = "a hive that does not host the swarm's shared services runs none of them"; + ok = + let + es = swarmServiceEnables bare; + in + lib.length (lib.attrNames es) == 9 && !lib.any lib.id (lib.attrValues es); + } + { + name = "a hive that has not opted into all-local runs no swarm controller"; + ok = !bare.services.hyperhive.deploy.swarm-controller.enable; + } + { + name = "the all-local mode turns the swarm controller on"; + ok = allLocal.services.hyperhive.deploy.swarm-controller.enable; + } + { + # Where the reader puts the files and where the daemon looks for them is + # one agreement spanning two modules. Asserted against the option rather + # than the literal so moving the directory moves both ends. + name = "hive-c0re is told where the agents' queue credential lands"; + ok = + allLocal.systemd.services.hive-c0re.environment.HIVE_C0RE_AGENT_QUEUE_CREDENTIAL_DIR + == toString allLocal.services.hyperhive.deploy.hive-controller.queue.agentCredentialDir; + } + { + # The one address in this file that must NOT be loopback. Both spellings + # sit in the same unit's environment and are correct for their own + # reader: hive-c0re shares the host netns, an agent container does not, + # so a copy-paste between them reaches the agent itself and the symptom + # is a connect that hangs. + name = "the agents' queue address is the bridge, not the loopback one the hive itself uses"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + in + e.HIVE_AGENT_NATS_URL == "nats://${allLocal.services.hyperhive.network.bridgeIp}:4222" + && !(lib.hasInfix "127.0.0.1" e.HIVE_AGENT_NATS_URL) + && e.HIVE_AGENT_NATS_URL != e.HIVE_C0RE_NATS_URL; + } + { + # The agents mint against the swarm's IdP, the same endpoint the hive's + # own client uses — a hive-local guess would produce a token the queue + # would not accept. + name = "the agents' token endpoint is the swarm IdP's"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + in + lib.hasSuffix "/api/oidc/token" e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT + && e.HIVE_AGENT_OIDC_TOKEN_ENDPOINT == e.HIVE_C0RE_OIDC_TOKEN_ENDPOINT; + } + { + # The absence arm, and what makes the two above able to fail: a hive + # with no queue address must forward neither coordinate, because half a + # pair reaches the harness as a partial configuration rather than as + # none. + name = "a hive with no swarm queue forwards no agent queue coordinates"; + ok = + let + e = bare.systemd.services.hive-c0re.environment; + in + !(e ? HIVE_AGENT_NATS_URL) && !(e ? HIVE_AGENT_OIDC_TOKEN_ENDPOINT); + } + { + # Where the store is and where the agent is told it is, one agreement + # spanning two modules. Asserted against the hive's own `BAO_ADDR` + # rather than a literal, because an agent pointed at a different + # spelling of the same store presents a certificate to a listener whose + # name it cannot verify. + name = "an agent is told the same store address its hive uses"; + ok = + let + e = allLocal.systemd.services.hive-c0re.environment; + in + e.HIVE_AGENT_BAO_ADDR == e.BAO_ADDR; + } + { + # A hive with no certificate of its own can collect no agent's identity, + # so forwarding an address would name a store nothing in the container + # can reach. The same gate the `BAO_*` pair beside it sits behind. + name = "a hive with no store identity forwards no store address to its agents"; + ok = !(bare.systemd.services.hive-c0re.environment ? HIVE_AGENT_BAO_ADDR); + } + { + # The gateway's per-name issuer choice. If this ever collapses to a + # constant, every swarm-service vhost serves a certificate its CA + # is name-constrained out of — which evaluates cleanly and fails in + # a browser. + name = "a swarm service name gets the swarm-services leaf and the default server does not"; + ok = + let + l = allLocal.services.hyperhive.gateway.lib; + in + (l.tlsFor "t.local").sslCertificate != (l.tlsFor "_").sslCertificate; + } + { + # nixos asserts when a vhost declares both, so this is also a + # statement that the `removeAttrs` upstream of it still happens. + name = "the swarm UI vhost forces TLS instead of merely adding it"; + ok = + let + v = allLocal.services.nginx.virtualHosts."t.local"; + in + v.forceSSL && !(v.addSSL or false); + } + { + name = "a hive with matrix off serves no matrix discovery endpoint"; + ok = + !(builtins.hasAttr "= /.well-known/matrix/client" bare.services.nginx.virtualHosts."_".locations); + } + { + # main got eval-borked twice by this exact class of bug (once on the + # unit's `Restart` key, once on `RestartSec`) — a nixpkgs bump to + # `gitea-actions-runner.nix` adds a plain `serviceConfig.*` + # definition that collides with one of ours, and nix refuses to + # merge two plain definitions at *host* eval. No other check + # instantiates a host with `containers.hive-ci` actually enabled, so + # the collision only surfaces on operator deploy, not in CI. + name = "the CI container's unit definitions merge without a priority collision"; + ok = forceCiServiceConfigs; + } + { + # The collector reaches these routes through the gateway now, so each + # store needs an ingest location of its own. Without one the write rides + # the `/` catch-all: unauthenticated on the metrics store, and into a + # browser redirect on the log store. + name = "each store's vhost has an authenticated ingest location"; + ok = + let + v = allLocal.services.nginx.virtualHosts; + m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push" or null; + l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs" or null; + in + m != null + && l != null + && lib.hasInfix "auth_request" m.extraConfig + && lib.hasInfix "auth_request" l.extraConfig; + } + { + # The arm that actually protects something. A pusher handed + # `error_page 401 =302` FOLLOWS it and POSTs its batch at a login page, + # which answers 200 — ingest reporting healthy while storing nothing. + # The third clause is the positive control: the log store's browser + # location really does redirect, so this says the machine routes differ + # rather than that the string is absent from the whole file. + name = "the ingest locations answer 401 instead of redirecting a pusher"; + ok = + let + v = allLocal.services.nginx.virtualHosts; + m = v."metrics.t.local".locations."= /opentelemetry/api/v1/push".extraConfig; + l = v."logs.t.local".locations."= /insert/opentelemetry/v1/logs".extraConfig; + browser = v."logs.t.local".locations."/".extraConfig; + in + !(lib.hasInfix "error_page" m) + && !(lib.hasInfix "error_page" l) + && lib.hasInfix "error_page" browser; + } + { + # The read counterpart to the ingest location: an agent queries the log + # store with a bearer token, and the `/` catch-all is the browser's + # route. Riding it would mean inheriting the login redirect the next + # case is about, so the route has to exist separately to be gated + # separately. + name = "the log store's vhost has an authenticated machine query location"; + ok = + let + q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/" or null; + in + q != null && lib.hasInfix "auth_request" q.extraConfig; + } + { + # Same trap as the ingest case, on the read side, where it is worse: a + # redirected pusher at least stores nothing visibly, while a redirected + # *reader* is handed a 200 carrying login HTML and records a query that + # succeeded and matched no logs. The browser clause is the positive + # control — that location really does redirect to the login host — so a + # pass means these two routes differ rather than that the strings are + # absent from the whole vhost. + name = "the machine query location answers 401 instead of redirecting to a login page"; + ok = + let + v = allLocal.services.nginx.virtualHosts; + q = v."logs.t.local".locations."^~ /select/logsql/".extraConfig; + browser = v."logs.t.local".locations."/".extraConfig; + in + !(lib.hasInfix "error_page" q) + && !(lib.hasInfix "auth.t.local" q) + && lib.hasInfix "error_page" browser + && lib.hasInfix "auth.t.local" browser; + } + { + # Read access is deliberately unscoped: an authenticated caller reads + # the whole swarm's logs until a permission system exists. Pinned so a + # scoping parameter arriving later is a visible diff here rather than a + # quiet change of rule — and pinned on `proxyPass` too, because + # VictoriaLogs takes its filters as request parameters, which ride an + # upstream URI as easily as a directive. The first clause is the + # control: it proves the location resolved and that `hasInfix` finds + # what is genuinely in this string, so the absences below mean absent + # rather than unreadable. + name = "the machine query location forwards the caller's query unmodified"; + ok = + let + q = allLocal.services.nginx.virtualHosts."logs.t.local".locations."^~ /select/logsql/"; + in + lib.hasInfix "auth_request" q.extraConfig + && !(lib.hasInfix "extra_filters" q.extraConfig) + && !(lib.hasInfix "extra_stream_filters" q.extraConfig) + && !(lib.hasInfix "$args" q.extraConfig) + && !(lib.hasInfix "?" q.proxyPass); + } + { + # The absence arm for the case above, and the option's own rule — a + # service declares its entry under its own `enable` — made checkable. + # Without it, moving the assignment outside the collector's `mkIf` + # passes every arm above while handing a collector-less hive a scrape + # target for a port nothing binds. + name = "a hive with no collector declares no self-scrape target"; + ok = bare.services.hyperhive.otel.scrapeTargets == { }; + } + { + # Absence arm, and the one that matters: claiming a name this host does + # not serve points every local reader at the wrong machine. + name = "a hive that does not run the store claims no name for it"; + ok = !(builtins.elem "bao.t.local" (baoNames bare)); + } + { + # ⚠️ The absence arm that matters. `services.nginx.streamConfig` is a + # host-wide option, so a block rendered outside the store's own `mkIf` + # gives every hive in the swarm a listener — on the port the store + # answers on, in front of no store at all. + name = "a hive that does not run the store renders no stream passthrough"; + ok = baoStream bare == ""; + } + { + # Absence arm for the case above — a hive with no store has no reason to + # open the store's port, and opening it would point agents at a host that + # answers nothing. + name = "a hive that does not run the store opens no bridge port for it"; + ok = !(builtins.elem 8200 (bridgePorts bare)); + } + { + # The four blocks below used to be gated on the hive being enabled AND + # their own condition. The second half was always the load-bearing one — + # none of these conditions is derived from the hive toggle — so these + # arms pin what the conjunct was doing: nothing. Each is written against + # a host with the hive OFF as well as one with it on, because the way a + # dropped conjunct fails is by making something unconditional, and that + # shows up as a service appearing where nothing asked for it. + name = "no bridge port is opened for an exposeHostPorts nobody set"; + ok = + !(builtins.elem 5432 (bridgePortsOrNone bare)) + && !(builtins.elem 5432 (bridgePortsOrNone centralToggleOff)); + } + { + # `deploy.swarm-controller.enable`, which defaults false and is + # deliberately not derived from the hive toggle — a swarm has one + # controller, so the host that runs it says so itself. + # + # Probed by what the daemon needs in order to run, not by + # `? swarm-controller`: ./host-modules/hive-tls.nix defines an + # environment key on that unit name, which leaves the attr + # present-but-inert (no `ExecStart`, empty `wantedBy`) on every hive + # that has a CA — see the comment there. The credential oneshot has no + # second definer, so its absence is the unambiguous half. + name = "the swarm controller does not run unless this host is told to run it"; + ok = + let + inert = + machine: + !(machine.systemd.services ? swarm-controller-credential) + && !( + (machine.systemd.services.swarm-controller or { serviceConfig = { }; }).serviceConfig ? ExecStart + ); + in + inert bare && inert centralToggleOff; + } + { + # `deploy.swarm-otel.enable`, same shape: the swarm's collector is one + # host's job, and the container is the whole of what it renders. + name = "the swarm collector container is absent unless this host is told to run it"; + ok = !(bare.containers ? swarm-otel) && !(centralToggleOff.containers ? swarm-otel); + } + { + # `deploy.swarm-ui.enable`, which defaults to the controller's toggle — + # derived from a sibling deployment decision, still not from the hive + # toggle. `t.local` is the fixtures' swarm domain, which is the apex the + # UI claims; the arm below is what proves this vhost renders at all. + name = "the swarm UI vhost is absent unless this host is told to serve it"; + ok = + !(bare.services.nginx.virtualHosts ? "t.local") + && !(centralToggleOff.services.nginx.virtualHosts ? "t.local"); + } + { + # The three infrastructure toggles are off by default and asserted by + # whoever needs them. With nothing on the host needing them, none of + # the three renders — which is also the control for the arm below. + name = "the gateway, resolver and bridge are absent where nothing on the host needs them"; + ok = + !centralToggleOff.services.hyperhive.gateway.enable + && !centralToggleOff.services.hyperhive.gateway.dns.enable + && !centralToggleOff.services.hyperhive.network.enable + && !(centralToggleOff.services.nginx.enable or false) + && !(centralToggleOff.services.dnsmasq.enable or false) + && !(centralToggleOff.networking.bridges ? hive-br0); + } + { + # hive-c0re asserts all three, and it follows the central toggle — so + # an ordinary hive keeps getting them with no opt-in, which is what + # this change must not break. + name = "an ordinary hive runs the gateway, resolver and bridge because its coordinator needs them"; + ok = + bare.services.hyperhive.gateway.enable + && bare.services.hyperhive.gateway.dns.enable + && bare.services.hyperhive.network.enable + && bare.services.nginx.enable + && bare.services.dnsmasq.enable + && bare.networking.bridges ? hive-br0; + } + ]; +in +runGroup "core-toggle" cases diff --git a/nix/module-eval/grafana.nix b/nix/module-eval/grafana.nix new file mode 100644 index 00000000..25442c56 --- /dev/null +++ b/nix/module-eval/grafana.nix @@ -0,0 +1,222 @@ +# `checks.module-eval-grafana` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + grafanaOldPath = hive { + deploy.grafana.enable = true; + swarm.grafana.socketDir = "/run/test-grafana-sock"; + swarm.grafana.datasourceUrl = "http://127.0.0.1:19999"; + swarm.grafana.logsDatasourceUrl = "http://127.0.0.1:19998"; + swarm.grafana.plugins = [ ]; + swarm.grafana.package = pkgs.emptyDirectory; + }; + + # The metrics UI beside the IdP. It reads its secret out of the store like + # every other Grafana host, so it needs a store identity like every other + # Grafana host — the cert pair here is not scenery, it is the arm that would + # have caught the deleted co-located copy unit coming back. + grafanaWithAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + + # The same UI with the IdP on ANOTHER host and a store leaf placed by hand. + # Knowing an IdP is not running one: `swarm.authelia.url` is what says this + # swarm has SSO, and nothing about this host does. Identical to the fixture + # above in everything the delivery path reads, which is the point. + grafanaRemoteAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + + # A Grafana host holding no store identity. This used to be the shape the + # module went QUIET on — no OIDC block, a warning, and a container whose + # login form is off regardless, so no way in and nothing failed. It is kept + # rather than deleted because the shape is still reachable by an operator; + # what changed is the deliverable, from a warning nothing reads back to a + # refusal naming the two options to set. Only the identity is missing, so an + # arm below can name which refusal fired. + grafanaNoIdentity = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + }; + + # The mirror image: the identity is placed, and the swarm names no IdP. The + # other half of "SSO must always be configured", and isolated the same way — + # exactly one thing wrong, so the arm reads one refusal. + grafanaNoSso = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + + # Did ./host-modules/swarm-grafana.nix refuse this host, and for which of its + # two reasons. An assertion is a config VALUE until something forces it — + # `.config` never throws — so a fixture in a state the module refuses is + # evaluable and the refusal is readable as data. That is what lets a case + # check that a misconfiguration is REPORTED, rather than only that it is not + # silently accepted. + # + # Matched on the option name the message names, not on its prose, so the + # wording stays rewordable: the option name is the part an operator has to + # act on, and a message that stopped naming it would be the actual defect. + grafanaRefusedFor = + m: option: + lib.any ( + a: + !a.assertion + && lib.hasInfix "services.hyperhive.deploy.grafana.enable requires" a.message + && lib.hasInfix option a.message + ) m.assertions; + cases = [ + { + # This fixture enables grafana and NOT authelia, which is the shape the + # login form used to stay enabled in: the toggle read "both services are + # on this host" rather than "grafana requires SSO". Grafana ships an + # `admin`/`admin` account and its vhost is on the public gateway, so a + # password box there is a way in whatever the topology. + name = "grafana disables its local login form even where authelia is not on this host"; + ok = + grafanaOldPath.containers.swarm-grafana.config.services.grafana.settings.auth.disable_login_form; + } + { + # The absence class this whole file is for, and the reported defect in one + # arm: the OIDC block hung off "authelia is on this host", so the split + # deployment got a Grafana with no SSO settings and no login form — no way + # in at all. The block is emitted in every deployment now, so the negative + # arm is not "no block elsewhere" but "the two do not name the same IdP": + # each host's block has to point at the URL the SWARM names, and a block + # built from `deploy.authelia` rather than `swarm.authelia.url` would pass + # a presence check on both fixtures while sending one of them nowhere. + name = "grafana's OIDC block names the swarm's IdP, wherever that IdP runs"; + ok = + let + oauth = m: m.containers.swarm-grafana.config.services.grafana.settings."auth.generic_oauth"; + remote = oauth grafanaRemoteAuthelia; + local = oauth grafanaWithAuthelia; + in + remote.enabled + && lib.hasInfix "https://auth.example.invalid/api/oidc/token" remote.token_url + && local.enabled + && lib.hasInfix "https://auth.t.local/api/oidc/token" local.token_url + && !(lib.hasInfix "auth.example.invalid" local.token_url); + } + { + # 🩸 The arm that guards the ruling this slice landed under. There is ONE + # delivery route: the store reader, on every host that runs Grafana. The + # negative names the deleted unit rather than a generic absence, because + # the way this regresses is someone re-adding the co-located copy as an + # optimisation — a second writer of one path, and a second shape of "the + # secret is wrong" to debug. + name = "grafana's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; + ok = + let + local = grafanaWithAuthelia.systemd.services; + remote = grafanaRemoteAuthelia.systemd.services; + in + local ? swarm-bao-grafana-oidc + && remote ? swarm-bao-grafana-oidc + && !(local ? swarm-grafana-oidc-secret) + && !(remote ? swarm-grafana-oidc-secret); + } + { + # What the deleted warning became. The shape is unchanged — a Grafana host + # holding no store leaf — but silence there is a container nobody can log + # into for a reason no log names, and a warning is read back by nothing. + # The second arm is what makes this a refusal about the IDENTITY: this + # fixture names an IdP, so a message about `swarm.authelia.url` here would + # mean the two assertions had been collapsed into one conjunction. + name = "a grafana host with no store identity is refused, naming the options to set"; + ok = + grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientCertFile" + && grafanaRefusedFor grafanaNoIdentity "deploy.bao.clientKeyFile" + && !(grafanaRefusedFor grafanaNoIdentity "swarm.authelia.url"); + } + { + # "SSO must always be configured", as an eval-time refusal rather than a + # gate. A null URL used to drop the OIDC block silently, and + # `disable_login_form` is unconditional a hundred lines below it, so that + # combination produced a Grafana with no SSO and no password box — an + # outage whose cause is a boolean that evaluated to false at build time + # and left no trace. Same isolation as the arm above, mirrored. + name = "a grafana host in a swarm with no IdP is refused, naming swarm.authelia.url"; + ok = + grafanaRefusedFor grafanaNoSso "services.hyperhive.swarm.authelia.url" + && !(grafanaRefusedFor grafanaNoSso "deploy.bao.clientCertFile"); + } + { + # Without this the two arms above prove nothing: a refusal that fires on + # every host is not a check, and both of these are hosts a swarm is + # expected to have. Read through the same helper, so a message that + # stopped naming its option would fail the arms above rather than pass + # this one by accident. + name = "neither grafana refusal fires on a correctly configured host, co-located or not"; + ok = + !(grafanaRefusedFor grafanaWithAuthelia "services.hyperhive.swarm.authelia.url") + && !(grafanaRefusedFor grafanaWithAuthelia "deploy.bao.clientCertFile") + && !(grafanaRefusedFor grafanaRemoteAuthelia "services.hyperhive.swarm.authelia.url") + && !(grafanaRefusedFor grafanaRemoteAuthelia "deploy.bao.clientCertFile"); + } + { + # Same 403-not-a-miss reason as the matrix and queue arms below: the + # reader's grant covers the `services` prefix, so a path outside it is + # refused rather than empty, however correct it reads. The negative arm is + # the rename this is exposed to — a secret filed under the hive that runs + # the service instead of under the service itself. + name = "grafana's OIDC secret is read from the prefix the publisher writes"; + ok = + let + s = grafanaRemoteAuthelia.systemd.services.swarm-bao-grafana-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # Both halves of the co-location assumption, which was one host's + # `deploy.*` answering a question about the whole swarm: the identities + # were minted only where the queue happened to run, and the token + # endpoint was known only where the IdP happened to run. + name = "hive identities and the token endpoint do not depend on which host runs what"; + ok = + let + autheliaNoQueue = hive { deploy.authelia.enable = true; }; + in + lib.elem "hive-h1" (map (c: c.id) autheliaNoQueue.services.hyperhive.swarm.authelia.oidc.clients) + && + grafanaRemoteAuthelia.services.hyperhive.swarm.statusPublish.tokenEndpoint + == "https://auth.example.invalid/api/oidc/token"; + } + ]; +in +runGroup "grafana" cases diff --git a/nix/module-eval/hive-otel.nix b/nix/module-eval/hive-otel.nix new file mode 100644 index 00000000..2d9c603d --- /dev/null +++ b/nix/module-eval/hive-otel.nix @@ -0,0 +1,129 @@ +# `checks.module-eval-hive-otel` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + otelSettings + ; + + # This hive's own collector, which is a HOST service — unlike the swarm + # tier's, which lives in a container and is read through `otelSettings`. + hiveOtel = hive { + otel.enable = true; + otel.clientSecretFile = "/var/lib/hive-otel-oidc/client.secret"; + }; + + hiveOtelPipelines = hiveOtel.services.opentelemetry-collector.settings.service.pipelines; + + hiveOtelSettings = hiveOtel.services.opentelemetry-collector.settings; + + # The hive tier's rendered scrape list. Same reasoning as `scrapeJob` for + # the swarm tier — the option is one string, what prometheus is handed is a + # job — but this collector is a host service, so the path to it differs. + hiveScrapeJob = + job: + lib.findFirst ( + c: c.job_name == job + ) null hiveOtelSettings.receivers.prometheus.config.scrape_configs; + # Duplicated from bao-otel-collector.nix — a case here needs it too. + + # The store with and without a collector on the same host. `scrapeTargets` + # is only ever read by a local collector, so the metrics endpoint is a + # function of the pairing rather than of the store. + baoWithCollector = hive { + deploy.bao.enable = true; + deploy.swarm-otel.enable = true; + # A second job declared as bare `host:port`, so the pair of cases below + # reads one rendered scrape list: the store's entry carries a path, this + # one carries none. + swarm.otel.scrapeTargets.plain = "127.0.0.1:9999"; + }; + cases = [ + { + # The tier in the middle. Its OTLP receiver takes both signals on one + # port, so without this pipeline an agent's push is answered 404 on + # `/v1/logs` — and a forwarder retrying into a 404 is indistinguish- + # able from one with nothing to send. Compared against the metrics + # pipeline's exporters rather than a name, so the two signals cannot + # drift to different destinations. + name = "the hive collector forwards logs upstream, not only metrics"; + ok = + (hiveOtelPipelines ? logs) + && hiveOtelPipelines.metrics.exporters != [ ] + && hiveOtelPipelines.logs.receivers == [ "otlp" ] + && hiveOtelPipelines.logs.exporters == hiveOtelPipelines.metrics.exporters; + } + { + # `deltatocumulative` is metrics-only: naming it in a logs pipeline + # kills the collector at startup rather than doing nothing. The second + # clause is the control — the metrics pipeline still names it, so a + # pass means the two processor lists differ rather than that the + # processor left the module. + name = "the hive collector keeps the metrics-only processor out of its logs pipeline"; + ok = + !(builtins.elem "deltatocumulative" hiveOtelPipelines.logs.processors) + && builtins.elem "deltatocumulative" hiveOtelPipelines.metrics.processors; + } + { + # The counters that say telemetry is being LOST — refused, failed, + # queue depth — are served on loopback and reach no store unless + # something reads them. Read off the rendered job rather than the + # option: only the job is what prometheus actually requests. + name = "the hive collector scrapes its own telemetry endpoint"; + ok = + let + j = hiveScrapeJob "collector"; + in + j != null && j.static_configs == [ { targets = [ "127.0.0.1:8888" ]; } ]; + } + { + # A `prometheus` receiver no pipeline names collects nothing while + # rendering and starting perfectly, so the scrape above is inert + # without this. The path is newly reachable: until the hive tier had a + # target of its own, this receiver was never emitted on any hive. + name = "the hive metrics pipeline names the prometheus receiver the self-scrape needs"; + ok = builtins.elem "prometheus" hiveOtelPipelines.metrics.receivers; + } + { + # `metrics.address` is the spelling that looks right and is rejected by + # this collector version. The first clause is the control: without it a + # missing telemetry block would pass the port check vacuously. + name = "the hive collector binds its telemetry port through readers, not address"; + ok = + let + m = hiveOtelSettings.service.telemetry.metrics; + in + !(m ? address) && (lib.head m.readers).pull.exporter.prometheus.port == 8888; + } + { + # Both collectors share a network namespace whenever they are + # co-located, and this port appears in no config the port-collision + # assertion can read — so equal defaults mean the second to start dies + # at `bind()`. Pinned as a case rather than an assertion: enforcing it + # belongs with the other port checks, not here. + name = "the two collector tiers do not claim the same self-telemetry port"; + ok = + let + portOf = s: (lib.head s.service.telemetry.metrics.readers).pull.exporter.prometheus.port; + in + portOf hiveOtelSettings != portOf (otelSettings baoWithCollector); + } + ]; +in +runGroup "hive-otel" cases diff --git a/nix/module-eval/lib.nix b/nix/module-eval/lib.nix new file mode 100644 index 00000000..016bfdd7 --- /dev/null +++ b/nix/module-eval/lib.nix @@ -0,0 +1,202 @@ +# `checks.module-eval-*` — the flake checks that cover **nix**. +# +# Split from one monolithic `module-eval` derivation into small +# independent checks, one per subsystem cluster, so that evaluating any +# single derivation only has to hold a handful of full `nixosSystem` +# fixtures live at once instead of all of them together — the original +# combined ~62 fixtures into one derivation and measured 10.6GB peak RSS / +# 5m25s to evaluate. Every other check in ./checks.nix is a Rust +# derivation, so a `.nix`-only diff moves no hash and `nix flake check` +# goes green **without evaluating what changed**; these derivations' hash +# is a function of their cases' results, so a change that flips a +# property rebuilds the owning check and fails naming it. +# +# Anything expressible as a module `assertion` **should be one instead** — +# an assertion fires at deploy time for a real operator, not only in CI. +# What cannot is the **absence class**: "a hive that hasn't opted in +# renders what it did before", "this unit does not exist unless X" — claims +# about the rendered config rather than about a config being invalid. +# +# ⚠️ **Name a case for the PROPERTY it defends, never the ticket that +# prompted it**; a case named after a ticket has the ticket's lifetime. +# +# ⚠️ **This evaluates, it does not execute** — a command line, a request or +# a certificate needs something that *runs* it. A case needing a rendered +# file must stub what it drags in (`deploy.swarm-ui.package = pkgs.emptyDirectory`). +# +# `hive`/`agent`/`agentWith` builders, the shared cross-group helpers, and +# the `runGroup` derivation builder live in ./lib.nix; each file below +# defines only the fixtures and cases its own cluster needs. +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + # Stub host, same shape ./docs/default.nix already uses: enough for a + # `nixosSystem` to evaluate, nothing that pulls a real disk or + # bootloader in. + hive = + extra: + (nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.default + { + fileSystems."/" = { + device = "/dev/null"; + fsType = "tmpfs"; + }; + boot.loader.grub.enable = false; + system.stateVersion = "25.11"; + # `recursiveUpdate`, not `//`: a plain `//` only merges the + # *top-level* keys of `extra` in, so an `extra` that touches a + # nested attr under an existing top-level key (e.g. `swarm.*`) + # silently drops every sibling under that key instead of merging + # into it — the same class of bug as the `services.hyperhive` + # double-nesting mistake this stub already had to dodge once, + # just one level down. `recursiveUpdate` merges nested attrsets + # all the way down instead. + services.hyperhive = lib.recursiveUpdate { + enable = true; + hiveName = "h1"; + swarm.domain = "t.local"; + swarm.hives.h1.domain = "h1.t.local"; + } extra; + } + ]; + }).config; + + # The other half of the tree. `nix/agent-modules/` is evaluated by nothing + # else in this suite — every fixture above is a host — so a rendered + # container config was only ever read by a real deploy. Same entry point + # the meta flake hands a container, so what this evaluates is what an + # agent gets. + # + # Takes a whole module rather than a settings attrset, so a fixture can + # reach either spelling of the agent tier. `user.name` is the one option + # with no usable default, set here at its real path and at `mkDefault` so a + # fixture naming its own agent still wins. + agentWith = + module: + (nixosSystem { + system = pkgs.stdenv.hostPlatform.system; + modules = [ + self.nixosModules.agent-base + { + fileSystems."/" = { + device = "/dev/null"; + fsType = "tmpfs"; + }; + boot.loader.grub.enable = false; + system.stateVersion = "25.11"; + services.hyperhive.agent.user.name = lib.mkDefault "a1"; + } + module + ]; + }).config; + + # Note `hyperhive`, not `services.hyperhive`: the agent tier's options moved + # under `services.hyperhive.agent`, and ../agent-modules/renamed-options.nix + # keeps the top-level spelling reaching them. ⚠️ That shim covers the + # options that existed when the tier moved and nothing since, so a fixture + # for an option added afterwards has to go through [`agentWith`] and name + # the real path. + agent = extra: agentWith { hyperhive = extra; }; + + baoNames = machine: machine.services.hyperhive.gateway.localNames; + + # What nginx is handed for its `stream {}` block. Deliberately not + # `virtualHosts`: a vhost is the terminating shape ./host-modules/ + # swarm-bao.nix's header refuses, and this is the passthrough that is not. + baoStream = machine: machine.services.nginx.streamConfig; + + # The bridge-interface firewall — the list `network.exposeHostPorts` merges + # into, and the only place a port is opened for agents. The host's own + # `allowedTCPPorts` is a different list and a different exposure. + bridgePorts = + machine: + machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName}.allowedTCPPorts; + + # Same list, on a host that may not declare the interface at all: the + # `[ 80 443 ]` block is what creates the attr, and it is off where the hive + # is. Reading it through `bridgePorts` would throw rather than report an + # empty exposure, which is precisely the state the cases below assert. + bridgePortsOrNone = + machine: + (machine.networking.firewall.interfaces.${machine.services.hyperhive.network.bridgeName} or { + allowedTCPPorts = [ ]; + } + ).allowedTCPPorts; + + # The config file openbao parses, not the nix that produces it: a setting it + # requires is absent here without anything in the module system minding, so + # the daemon's own startup is otherwise the first reader. + baoSettings = machine: machine.containers.swarm-bao.config.services.openbao.settings; + + otelSettings = + machine: machine.containers.swarm-otel.config.services.opentelemetry-collector.settings; + + agentHarness = machine: machine.systemd.services.hive-agent; + + # The enables that switch owns. ⚠️ `otel` is the per-hive collector's own + # option and is NOT under `deploy` — spelled at the wrong path it would be + # undeclared rather than false, and a roster that quietly loses a member is + # what the count guard in the cases below exists to catch. Its membership + # here is deliberate and was the fix for a gap, not an oversight: the hive + # tier lands wherever the swarm services do. + swarmServiceEnables = + machine: + let + h = machine.services.hyperhive; + in + { + matrix = h.deploy.matrix.enable; + otel = h.otel.enable; + authelia = h.deploy.authelia.enable; + nats = h.deploy.nats.enable; + swarm-otel = h.deploy.swarm-otel.enable; + victoriametrics = h.deploy.victoriametrics.enable; + grafana = h.deploy.grafana.enable; + victorialogs = h.deploy.victorialogs.enable; + bao = h.deploy.bao.enable; + }; + runGroup = + name: cases: + let + bad = builtins.filter (c: !c.ok) cases; + # Escaped, because a name is prose and prose contains apostrophes. Hand-quoting + # broke the builder mid-report on the first such name that failed — and only + # ever on failure, so every green run agreed the reporter was fine. + report = lib.concatMapStringsSep "\n" ( + c: " echo ${lib.escapeShellArg "FAILED: ${c.name}"} >&2" + ) bad; + in + # The results are embedded in the builder text on purpose: that is what + # makes this derivation's hash depend on them, so a nix-only change that + # flips a case cannot be answered from cache. + pkgs.runCommand "hyperhive-module-eval-${name}" { } '' + ${report} + ${ + if bad == [ ] then + "echo '${toString (builtins.length cases)} module properties hold in ${name}' && touch $out" + else + "echo 'module-eval-${name}: ${toString (builtins.length bad)} of ${toString (builtins.length cases)} properties broke' >&2 && exit 1" + } + ''; +in +{ + inherit hive; + inherit agent; + inherit agentWith; + inherit baoNames; + inherit baoStream; + inherit bridgePorts; + inherit bridgePortsOrNone; + inherit baoSettings; + inherit otelSettings; + inherit agentHarness; + inherit swarmServiceEnables; + inherit runGroup; +} diff --git a/nix/module-eval/matrix-core.nix b/nix/module-eval/matrix-core.nix new file mode 100644 index 00000000..d5ef8f17 --- /dev/null +++ b/nix/module-eval/matrix-core.nix @@ -0,0 +1,79 @@ +# `checks.module-eval-matrix-core` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # A deliberately odd `maxRequestSize`, so the number the assertion looks for + # cannot have come from a default or from another module's literal. + matrixBodyCap = hive { + deploy.singleHostSwarm = true; + deploy.matrix.maxRequestSize = 99000000; + }; + + # The same hive with the identity taken away, which separates "a homeserver + # is deployed" from "this host can authenticate to the store". + matrixNoBaoIdentity = hive { deploy.matrix.enable = true; }; + cases = [ + { + # Absence arm, and what makes the one above able to fail: with no leaf + # this unit would fail a TLS handshake on every boot, so it must not + # exist at all rather than retry its way through the start limit. + name = "a hive with no store identity renders no queue credential reader"; + ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-queue-agent); + } + { + # Two limits governed a matrix upload and nothing kept them in agreement, + # so raising the documented one past the gateway's hardcoded cap changed + # nothing. The second clause is the control: the old directive is gone, so + # a pass means the value is derived rather than that `hasInfix` matched + # something incidental. It targets the whole directive rather than the + # bare size, because the rendered config carries the comment above it and + # a prose mention of the old value would defeat a looser arm. + name = "the matrix gateway's body cap is derived from maxRequestSize, not a literal"; + ok = + let + c = matrixBodyCap.services.nginx.virtualHosts."chat.t.local".locations."/_matrix/".extraConfig; + in + lib.hasInfix "client_max_body_size 100048576;" c && !(lib.hasInfix "client_max_body_size 50M" c); + } + { + # Absence arm for the one above, and the reason the gate is the identity + # rather than the homeserver: without it, deploying matrix anywhere would + # render a reader that cannot authenticate. + name = "a homeserver with no way to authenticate to the store renders no token reader"; + ok = !(matrixNoBaoIdentity.systemd.services ? swarm-bao-matrix-token); + } + { + # Absence arm for the case above. A hive with no client identity gets no + # store environment at all — the daemon reports a queue it cannot serve + # rather than a handshake it cannot explain. + name = "a hive with no client identity gives hive-c0re no store environment"; + ok = + let + s = matrixNoBaoIdentity.systemd.services; + in + s ? hive-c0re + && !(s.hive-c0re.environment ? BAO_ADDR) + && !(lib.any (c: lib.hasPrefix "bao-" c) s.hive-c0re.serviceConfig.LoadCredential); + } + ]; +in +runGroup "matrix-core" cases diff --git a/nix/module-eval/name-guards.nix b/nix/module-eval/name-guards.nix new file mode 100644 index 00000000..d2ee8b32 --- /dev/null +++ b/nix/module-eval/name-guards.nix @@ -0,0 +1,115 @@ +# `checks.module-eval-name-guards` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # The hive-name guards, with the collector explicitly OFF. That is the whole + # property: the guards live where `swarm.hives` is declared, so they run in a + # deployment that has a secret store and no collector — which used to skip + # them entirely, because they were assertions inside swarm-otel's own `mkIf`. + # + # ⚠️ `controllerCommonName` is overridden to a name containing NO reserved + # fragment. Its default (`swarm-controller`) contains `swarm` and is caught + # by the substring guard whatever the cert-auth arm does — so a fixture using + # the default could not tell the two apart, and the arm under test would pass + # on the neighbour's work. + hiveNamedAfterCertSubject = hive { + deploy.swarm-otel.enable = false; + deploy.bao.controllerCommonName = "ctl"; + swarm.hives.ctl.domain = "ctl.t.local"; + }; + + # The control for both arms below: same shape, a roster nothing objects to. + hiveNamesAllLegal = hive { + deploy.swarm-otel.enable = false; + deploy.bao.controllerCommonName = "ctl"; + }; + + # The reserved subjects are a LIST, and a list with one consulted element and + # one dead one looks identical from the first element's case. This fixture + # collides with the SECOND, leaving the controller's at its default. + hiveNamedAfterPublisherSubject = hive { + deploy.swarm-otel.enable = false; + deploy.bao.secretPublisherCommonName = "pubctl"; + swarm.hives.pubctl.domain = "p.t.local"; + }; + + hiveNameWithComposedWord = hive { + deploy.swarm-otel.enable = false; + swarm.hives."h1-agent".domain = "a.t.local"; + }; + + # Markers from `lib/name-guards.nix`'s two `problem` strings. Matching the + # problem rather than the `why` prose keeps the messages rewordable. + equalityGuardFired = + h: lib.any (a: !a.assertion && lib.hasInfix "has reserved name(s)" a.message) h.assertions; + + fragmentGuardFired = + h: + lib.any ( + a: !a.assertion && lib.hasInfix "has name(s) containing a reserved word" a.message + ) h.assertions; + cases = [ + { + # `ctl` is in no deny list — it is reserved *because it is the subject a + # cert-auth role accepts*, which is a value an operator sets, so a + # literal deny entry could never have covered it. + name = "a hive named after a cert-auth subject is refused, with the collector off"; + ok = + equalityGuardFired hiveNamedAfterCertSubject + && lib.any (a: !a.assertion && lib.hasInfix "'ctl'" a.message) hiveNamedAfterCertSubject.assertions; + } + { + # Every cert-auth subject is reserved, not just the first one in the + # list. Without this case the second element could be dead and the case + # above would still pass. + name = "a hive named after the secret publisher's subject is refused too"; + ok = + equalityGuardFired hiveNamedAfterPublisherSubject + && lib.any ( + a: !a.assertion && lib.hasInfix "'pubctl'" a.message + ) hiveNamedAfterPublisherSubject.assertions; + } + { + # Without this the case above proves nothing: an arm that fires for every + # roster is not a guard, and `hives` is non-empty in both fixtures. + name = "a legal hive roster trips neither name guard"; + ok = !(equalityGuardFired hiveNamesAllLegal) && !(fragmentGuardFired hiveNamesAllLegal); + } + { + # The substring guard came along in the move and has to still work. + # `h1-agent` mints exactly the client id hive `h1`'s agents present. + name = "a hive name containing a composed-identifier word is refused, with the collector off"; + ok = fragmentGuardFired hiveNameWithComposedWord; + } + { + # ⚠️ The control that makes "with the collector off" mean anything. If a + # fixture silently had swarm-otel enabled, all three cases above would + # pass while testing the arrangement they exist to rule out. + name = "the guard fixtures really do have the collector disabled"; + ok = + !hiveNamedAfterCertSubject.services.hyperhive.deploy.swarm-otel.enable + && !hiveNamesAllLegal.services.hyperhive.deploy.swarm-otel.enable + && !hiveNameWithComposedWord.services.hyperhive.deploy.swarm-otel.enable; + } + ]; +in +runGroup "name-guards" cases diff --git a/nix/module-eval/nats-authelia.nix b/nix/module-eval/nats-authelia.nix new file mode 100644 index 00000000..4f343c04 --- /dev/null +++ b/nix/module-eval/nats-authelia.nix @@ -0,0 +1,121 @@ +# `checks.module-eval-nats-authelia` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # The queue's callout identity, fourth split slice. `autoGenerateCallout` is + # left FALSE on purpose: that is what makes the seed paths the thing deciding + # `responderConfigured`, so the assertion below is about the seeds rather + # than about the auto-mint branch. Every one of the seven old paths is + # defined — `enable` included, which is why it is spelled the old way here + # while the fixture below uses the new one — so dropping any single nats + # shim fails the eval, not just the arms read. + natsOldPath = hive { + swarm.nats.enable = true; + swarm.nats.autoGenerateCallout = false; + swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + swarm.nats.authPackage = pkgs.emptyDirectory; + }; + + # Seventh split slice, plus slice 10's two authelia packages. Of slice 7's + # movers only `usersFile` has a rename entry — the other two are `readOnly`, + # and a rename module contributes a definition, which a read-only option + # refuses; see ./host-modules/deploy.nix. `package` and `bridgePackage` are + # ordinary options, so they do carry one. The two arms + # below have different jobs. `usersFile` tests the rename; the nats one tests + # that a reader repointed to the new namespace still renders the derived + # path, which is the failure this slice could actually have shipped — seven + # of those reads went through an alias a path-shaped grep cannot see. + autheliaOldPath = hive { + deploy.authelia.enable = true; + deploy.nats.enable = true; + swarm.authelia.usersFile = "/var/lib/test-authelia/users.yml"; + swarm.authelia.package = pkgs.emptyDirectory; + swarm.authelia.bridgePackage = pkgs.emptyDirectory; + swarm.nats.autoGenerateCallout = false; + swarm.nats.calloutUserPublicKey = "UTESTUSERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutIssuerPublicKey = "ATESTISSUERPUBKEYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + swarm.nats.calloutUserSeedFile = "/run/secrets/nats-user.seed"; + swarm.nats.calloutIssuerSeedFile = "/run/secrets/nats-issuer.seed"; + }; + cases = [ + { + # Reads the RENDERED settings, not the option: `calloutBlocks {…} // { + # … }` is a shallow merge, and a future edit that dropped or shadowed + # this key would still evaluate cleanly — the only reader that would + # notice is a publisher whose row exceeds upstream's much smaller + # default, and by then it is a dropped row, not an eval failure. + # Piggybacks on the pre-rename nats fixture above, which already + # renders this container's full config. + name = "the queue's payload ceiling is set, not inherited from the server's default"; + ok = natsOldPath.containers.swarm-nats.config.services.nats.settings.max_payload == 8388608; + } + { + # Reads the RENDERED unit text, not the module's source, because the + # failure this defends against renders perfectly: systemd substitutes + # `$NAME` in `ExecStart` regardless of quoting, so a single dollar + # here hands the responder `.term.{hive}.>` — a grant that parses, is + # accepted, and matches nothing an agent ever publishes to. Asserting + # the doubled dollar is the only way to tell the two apart before + # deploy. The flag's presence is asserted separately so that dropping + # the grant entirely fails as its own arm rather than as an escaping + # complaint. + name = "the responder grants agents their hive's terminal subject, and the dollar survives systemd"; + ok = + let + exec = + natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart; + in + lib.hasInfix "--agent-publish-subject " exec && lib.hasInfix "$$SWARM.term.{hive}.>" exec; + } + { + # Second grant, same escaping trap, asserted separately: the two + # subject families are independent features (terminal rows and the + # turn-state header) and dropping either should fail as its own arm + # rather than being masked by the other still being present. + # + # Flag and argument are matched as one infix rather than as two + # independent `hasInfix` calls: the responder takes the flag + # repeatedly, so the thing worth pinning is that THIS subject is the + # argument of one of them, which two separate presence checks would + # both pass on while the subject sat under some other flag entirely. + name = "the responder grants agents their hive's agent-state subject too"; + ok = + let + exec = + natsOldPath.containers.swarm-nats.config.systemd.services.swarm-nats-auth.serviceConfig.ExecStart; + in + lib.hasInfix "--agent-publish-subject '$$SWARM.agent-state.{hive}.>'" exec; + } + { + # Not a rename test. `hostClientSecretDir` is `readOnly`, so the fixture + # cannot define it; what can break is a reader left pointing at the + # namespace it moved out of. Five modules read this through an + # `autheliaCfg` alias, where a path-shaped grep does not see it. + name = "a consumer of authelia's host client-secret dir renders it from the deploy namespace"; + ok = lib.hasInfix "/var/lib/nixos-containers/swarm-authelia/var/lib/authelia-swarm/oidc-clients/" autheliaOldPath.systemd.services.swarm-nats-auth-secrets.script; + } + ]; +in +runGroup "nats-authelia" cases diff --git a/nix/module-eval/secret-publisher.nix b/nix/module-eval/secret-publisher.nix new file mode 100644 index 00000000..7c2f9f6d --- /dev/null +++ b/nix/module-eval/secret-publisher.nix @@ -0,0 +1,302 @@ +# `checks.module-eval-secret-publisher` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + ; + + # The IdP and the store on one machine: the shape where minted plaintext and + # a store identity are both present without an operator placing anything. + # Two hives in the roster, because the publisher walks it — an arm written + # against a single-hive fixture passes on a hardcoded name. + secretPublisherHere = hive { + deploy.bao.enable = true; + deploy.authelia.enable = true; + swarm.hives.h2.domain = "h2.t.local"; + }; + + # The IdP with no store on the box and a leaf placed by hand, which is the + # deployment this unit exists for: authelia is the one host the store is + # guaranteed not to share once either has a machine of its own. + # + # ⚠️ `enable` is deliberately NOT set here. It used to be, with a comment + # saying the default asked whether both ran on this host — which documented + # the co-location bug instead of catching it. Leaving it unset is what makes + # this fixture exercise the default rather than mask it. + secretPublisherRemote = hive { + deploy.authelia.enable = true; + deploy.swarm-secret-publisher.baoClientCertFile = "/etc/pki/publisher.pem"; + deploy.swarm-secret-publisher.baoClientKeyFile = "/etc/pki/publisher-key.pem"; + }; + + # The same IdP with the identity taken away. Minting the secrets is not being + # able to publish them, and this is the arm that separates the two. + secretPublisherNoIdentity = hive { deploy.authelia.enable = true; }; + # Duplicated from grafana.nix — a case here needs it too. + + # The same UI with the IdP on ANOTHER host and a store leaf placed by hand. + # Knowing an IdP is not running one: `swarm.authelia.url` is what says this + # swarm has SSO, and nothing about this host does. Identical to the fixture + # above in everything the delivery path reads, which is the point. + grafanaRemoteAuthelia = hive { + deploy.grafana.enable = true; + deploy.grafana.plugins = [ ]; + deploy.grafana.package = pkgs.emptyDirectory; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + # Duplicated from swarm-otel-core.nix — a case here needs it too. + + # The same collector with the IdP on ANOTHER host and a store leaf placed by + # hand. Identical to the fixture above in everything the delivery path + # reads, which is the point. + otelBaoRemoteAuthelia = hive { + deploy.swarm-otel.enable = true; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + # Duplicated from core-toggle.nix — a case here needs it too. + + bare = hive { }; + # Duplicated from bao-matrix-reader.nix — a case here needs it too. + + # The store and a service that reads from it, versus the store alone. The + # pair is what makes the reader's absence arm mean anything. + baoWithMatrix = hive { + deploy.bao.enable = true; + deploy.matrix.enable = true; + }; + cases = [ + { + # Both ends of a wire nothing at eval time carries end to end: the + # publisher on authelia's host writes the path the reader on Grafana's host + # reads, and the two files agree only because both compose it from the same + # swarm-wide client id. + name = "the publisher writes the swarm service path grafana reads"; + ok = lib.hasInfix "secret/swarm/services/swarm-grafana/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } + { + # Registering the client cannot live where the rest of grafana's module + # lives: that block is gated on this host RUNNING grafana, so on the split + # deployment nothing registered the client, authelia minted no secret, and + # every layer below had nothing to carry. The second arm is the control — + # a host with no IdP registers nothing. + name = "the swarm's grafana client is registered wherever authelia runs"; + ok = + let + clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; + in + lib.elem "swarm-grafana" (clients secretPublisherHere) + && !(lib.elem "swarm-grafana" (clients grafanaRemoteAuthelia)); + } + { + # The collector's half of the same defect and the same fix: this used to + # be gated on `deploy.swarm-otel.enable`, so a split deployment + # registered the client nowhere and authelia minted nothing to publish. + name = "the swarm's collector client is registered wherever authelia runs"; + ok = + let + clients = m: map (c: c.id) m.services.hyperhive.swarm.authelia.oidc.clients; + in + lib.elem "swarm-collector" (clients secretPublisherHere) + && !(lib.elem "swarm-collector" (clients otelBaoRemoteAuthelia)); + } + { + # Both ends of a wire nothing at eval time carries end to end: the + # publisher on authelia's host writes the path the reader on the + # collector's host reads, and the two files agree only because both + # compose it from the same swarm-wide client id. `secretPublisherHere` + # already grew this client when `serviceClientIds` did. + name = "the publisher writes the swarm service path the collector reads"; + ok = lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" ( + secretPublisherHere.systemd.services.swarm-secret-publish.script + ); + } + { + # The same hole the controller's case above names, open a second time: the + # PKI script grew a third leaf and no case read it. + name = "the store mints a leaf for the secret publisher, and the publisher is pointed at it"; + ok = + let + m = secretPublisherHere; + p = m.services.hyperhive.deploy.swarm-secret-publisher; + in + lib.hasInfix "secret-publisher.pem" m.systemd.services.swarm-bao-pki.script + && p.baoClientCertFile == "/var/lib/swarm-bao-pki/secret-publisher.pem" + && p.baoClientKeyFile == "/var/lib/swarm-bao-pki/secret-publisher-key.pem"; + } + { + # mara caught this by reading, which means no arm existed for it: the + # default asked `authelia.enable && bao.enable`, so the split deployment + # this unit is FOR defaulted off and published nothing, silently. + # + # The second clause is the control. Without it this passes on a default + # of plain `true`, which would be a different bug with the same symptom + # — an IdP-less host claiming it publishes secrets it never mints. + name = "the publisher defaults on where secrets are minted, whether or not the store is local"; + ok = + secretPublisherRemote.services.hyperhive.deploy.swarm-secret-publisher.enable + && !bare.services.hyperhive.deploy.swarm-secret-publisher.enable; + } + { + # The one security property of this unit, and why its push cannot be + # rewritten into the obvious shape: `bao` is an external binary, so an + # argument is world-readable in /proc for the life of the call. + # `value=@` hands it the path and bao opens the file itself. + # + # The second arm is what makes the first mean anything — `value=@` can + # sit one line above a command substitution that put the plaintext in + # argv anyway. + # + # ⚠️ Comments are stripped first, and that is not tidiness. A `script` + # renders its own comments into the text, and this unit's comments name + # the hazard verbatim so the next editor does not reintroduce it. Without + # the strip this case reads that warning and fails — a check the artifact + # defeats by DESCRIBING the thing it is checked for. + name = "the publisher hands bao the secret's path, never the secret"; + ok = + let + s = secretPublisherHere.systemd.services.swarm-secret-publish.script; + code = lib.concatStringsSep "\n" ( + lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s) + ); + in + lib.hasInfix "value=@" code && !(lib.hasInfix "$(cat" code); + } + { + # Two ends of a wire nothing at eval time carries end to end: this is the + # path `swarm_secret_client::queue` resolves for the reader. Both hives + # are asserted, so a publisher that knew one name rather than the roster + # fails here rather than on the second hive ever added to a swarm. + name = "the publisher writes every hive in the roster to that hive's own queue path"; + ok = + let + s = secretPublisherHere.systemd.services.swarm-secret-publish.script; + in + lib.hasInfix "secret/swarm/hives/h1/queue/agent" s + && lib.hasInfix "secret/swarm/hives/h2/queue/agent" s; + } + { + # The producer's end of the read `glue-matrix-bao-token.nix` already did. + # Both hives are asserted for the reason the queue case above gives: a + # publisher that knew one name rather than the roster would pass on a + # single-hive fixture and strand the second hive ever added — which is + # the two-hives-never-converge shape this slice exists to close. + name = "the publisher mints an appservice token for every hive and writes it to that hive's matrix path"; + ok = + let + s = secretPublisherHere.systemd.services.swarm-secret-publish.script; + in + lib.hasInfix "secret/swarm/hives/h1/matrix/appservice-token" s + && lib.hasInfix "secret/swarm/hives/h2/matrix/appservice-token" s + && lib.hasInfix "/dev/urandom" s; + } + { + # What makes a re-publish idempotent. This principal is granted + # `create`/`update` and no `read`, so it cannot ask the store whether a + # hive already has a token — with nowhere to keep one, every run would + # mint a fresh value and rotate the swarm's token. A state directory is + # that somewhere, and nothing else in this unit needs one, so its absence + # means exactly this. + # + # The second arm is the mint's own guard: the state file is only written + # when it is missing or empty. Dropping that test leaves a unit that + # still has a state directory and still rotates on every boot. + name = "the publisher keeps the tokens it minted, and mints only when it holds none"; + ok = + let + u = secretPublisherHere.systemd.services.swarm-secret-publish; + in + lib.hasInfix "matrix-appservice-token" (u.serviceConfig.StateDirectory or "") + && u.serviceConfig.StateDirectoryMode or null == "0700" + && lib.hasInfix "if [ ! -s \"$src\" ]" u.script; + } + { + # A property of the SET, not of one unit: both of these authenticate by + # certificate, and `BAO_CLIENT_CERT` is transport rather than identity, so + # a script that reaches `bao kv` without a token asks a token helper this + # host does not carry and fails before the store ever answers. `-token-only` + # is what keeps the token off the helper on the way back out. + # + # Ordering, not presence: the login has to come first, so the check is + # that nothing before it is a data command. Comments are stripped because + # both units explain this in prose directly above the code. + name = "the cert-identity bao units log in before their first read or write, and keep the token out of the helper"; + ok = + let + code = + s: + lib.concatStringsSep "\n" ( + lib.filter (l: builtins.match "[[:space:]]*#.*" l == null) (lib.splitString "\n" s) + ); + holdsTokenFirst = + s: + let + c = code s; + in + lib.hasInfix "bao login" c + && lib.hasInfix "-token-only" c + && !(lib.hasInfix "bao kv" (lib.head (lib.splitString "bao login" c))); + in + holdsTokenFirst secretPublisherHere.systemd.services.swarm-secret-publish.script + && holdsTokenFirst baoWithMatrix.systemd.services.swarm-bao-matrix-token.script + # Controls, so a clean verdict above means something. In order: a bare + # read is refused, a read placed before the login is refused, and a + # login that exists only in a comment is refused — that last one is the + # arm the comment-stripping exists for. + && !(holdsTokenFirst "bao kv get -field=value secret/x") + && !(holdsTokenFirst "bao kv get secret/x\nBAO_TOKEN=\"$(bao login -method=cert -token-only)\"") + && !(holdsTokenFirst "# bao login -method=cert -token-only goes here\nbao kv get secret/x") + && holdsTokenFirst "BAO_TOKEN=\"$(bao login -method=cert -token-only)\"\nbao kv get secret/x"; + } + { + # The doctrine three glue files state, as a property a rewrite has to + # keep: a client is defined by holding a certificate the store accepts, + # never by standing next to the store. Gating this on `deploy.bao.enable` + # would have left the unit rendering only on the one deployment that has + # no use for it. + name = "a publisher holding an identity runs on a host with no store"; + ok = + let + m = secretPublisherRemote; + in + !m.services.hyperhive.deploy.bao.enable + && (m.systemd.services ? swarm-secret-publish) + && (m.systemd.paths ? swarm-secret-publish); + } + { + # What makes the arm above able to fail. Minting the secrets is not being + # able to publish them: with no certificate the unit would fail a TLS + # handshake on every rotation, so it must not exist at all. + name = "an IdP host with no store identity renders no publisher"; + ok = + let + m = secretPublisherNoIdentity; + in + m.services.hyperhive.deploy.authelia.enable + && !(m.systemd.services ? swarm-secret-publish) + && !(m.systemd.paths ? swarm-secret-publish); + } + ]; +in +runGroup "secret-publisher" cases diff --git a/nix/module-eval/swarm-otel-core.nix b/nix/module-eval/swarm-otel-core.nix new file mode 100644 index 00000000..c4433f39 --- /dev/null +++ b/nix/module-eval/swarm-otel-core.nix @@ -0,0 +1,160 @@ +# `checks.module-eval-swarm-otel-core` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + otelSettings + ; + + # A swarm collector on a host that runs NEITHER store — the fully-spread + # shape from docs/swarm/services.md, and the one the old per-host gates made + # inexpressible. It is the whole point of the cases below that this hive is + # not a degenerate configuration but a supported one. + otelNoStores = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + deploy.victoriametrics.enable = false; + deploy.victorialogs.enable = false; + }; + + # The collector beside authelia, reading its own OIDC secret out of the + # store like every other collector — the cert pair here is not scenery, it + # is the arm that would catch the deleted co-located copy unit coming back. + otelBaoWithAuthelia = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + + # The same collector with the IdP on ANOTHER host and a store leaf placed by + # hand. Identical to the fixture above in everything the delivery path + # reads, which is the point. + otelBaoRemoteAuthelia = hive { + deploy.swarm-otel.enable = true; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.bao.clientCertFile = "/etc/pki/bao-client.pem"; + deploy.bao.clientKeyFile = "/etc/pki/bao-client-key.pem"; + }; + cases = [ + { + # 🩸 The arm that guards the ruling this slice landed under, the + # collector's half of ./swarm-grafana.nix's own. There is ONE delivery + # route: the store reader, on every host that runs the collector and + # holds a store identity. The negative names the deleted unit rather + # than a generic absence, because the way this regresses is someone + # re-adding the co-located copy as an optimisation. + name = "the collector's OIDC secret has exactly one delivery unit, the store reader, in both topologies"; + ok = + let + local = otelBaoWithAuthelia.systemd.services; + remote = otelBaoRemoteAuthelia.systemd.services; + in + local ? swarm-bao-otel-oidc + && remote ? swarm-bao-otel-oidc + && !(local ? swarm-otel-oidc-secret) + && !(remote ? swarm-otel-oidc-secret); + } + { + # Same 403-not-a-miss reason as grafana's arm above: the reader's grant + # covers the `services` prefix, so a path outside it is refused rather + # than empty, however correct it reads. + name = "the collector's OIDC secret is read from the prefix the publisher writes"; + ok = + let + s = otelBaoRemoteAuthelia.systemd.services.swarm-bao-otel-oidc.script; + in + lib.hasInfix "secret/swarm/services/swarm-collector/oidc/client" s + && !(lib.hasInfix "secret/swarm/hives/" s); + } + { + # The defect itself. These exporters used to be gated on the stores' + # PER-HOST enables, so a collector that did not share a host with them + # rendered none at all and dropped everything it received, from every + # hive — silently, because an absent exporter is not an error. + name = "a collector that hosts neither store still exports to both"; + ok = + let + e = (otelSettings otelNoStores).exporters; + in + (e ? "otlphttp/victoriametrics") && (e ? "otlphttp/victorialogs"); + } + { + # A swarm has one of each store, so the address is a swarm-level name. + # A loopback literal here is the co-location assumption written back in, + # and it renders, deploys and reports healthy while reaching nothing. + name = "the store exporters address the stores by name, never by loopback"; + ok = + let + e = (otelSettings otelNoStores).exporters; + m = e."otlphttp/victoriametrics".metrics_endpoint; + l = e."otlphttp/victorialogs".logs_endpoint; + in + !(lib.hasInfix "127.0.0.1" m) + && !(lib.hasInfix "127.0.0.1" l) + && lib.hasInfix "metrics.t.local" m + && lib.hasInfix "logs.t.local" l; + } + { + # `_HOSTNAME` cannot separate machines on its own: a hostname is a + # config value two of them can share, and then every stream for a unit + # name merges into one. + name = "the log stream is keyed by machine, not only by a hostname every container shares"; + ok = + let + l = (otelSettings otelNoStores).exporters."otlphttp/victorialogs".logs_endpoint; + field = f: lib.hasInfix ("_stream_fields=" + f) l || lib.hasInfix ("," + f) l; + in + field "_MACHINE_ID" && field "_SYSTEMD_UNIT" && !(field "_NOSUCHFIELD"); + } + { + # Defining an exporter and REFERENCING it are two separate lists, and + # the second is where the original gate also lived. An exporter no + # pipeline names is as silent as one that does not exist — this case + # exists because a mutation that restored only the reference-side gate + # left every other case here green. + name = "every pipeline that has a store exporter defined actually sends to it"; + ok = + let + s = otelSettings otelNoStores; + used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues s.service.pipelines)); + in + builtins.elem "otlphttp/victoriametrics" used && builtins.elem "otlphttp/victorialogs" used; + } + { + # An authenticator an exporter names but `service.extensions` omits is + # INERT — the collector starts clean and pushes unauthenticated until + # something at the far end refuses it. Checked as a set relation rather + # than by naming the two, so it keeps holding for exporters not written + # yet. + name = "every exporter authenticator is listed in service.extensions"; + ok = + let + s = otelSettings otelNoStores; + named = lib.filter (v: v != null) ( + lib.mapAttrsToList (_: e: e.auth.authenticator or null) s.exporters + ); + in + named != [ ] && lib.all (a: builtins.elem a s.service.extensions) named; + } + ]; +in +runGroup "swarm-otel-core" cases diff --git a/nix/module-eval/swarm-otel-identity.nix b/nix/module-eval/swarm-otel-identity.nix new file mode 100644 index 00000000..67c89357 --- /dev/null +++ b/nix/module-eval/swarm-otel-identity.nix @@ -0,0 +1,167 @@ +# `checks.module-eval-swarm-otel-identity` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + otelSettings + ; + + # A collector holding no store identity at all. Unlike Grafana's mirror + # image, this is not a refused shape: the collector still receives every + # hive's telemetry with nothing to push authenticated with, which is the + # already-supported degrade `haveCollectorSecret` names above the module's + # `let`. What this fixture is for is checking the reading unit itself does + # not render, rather than rendering with an env var nothing filled in. + otelNoIdentity = hive { + deploy.swarm-otel.enable = true; + swarm.authelia.url = "https://auth.example.invalid"; + deploy.forgejo.sso.clientSecretFile = "/var/lib/forgejo-oidc/by-hand.secret"; + }; + + # authelia somewhere else, the credential delivered by hand. Whether this + # collector authenticates must follow the credential, never another + # service's placement. + # + # The `swarm.otel.clientSecretFile` below is the PRE-RENAME path. It predates + # the split and is deliberately left spelled that way: it makes this fixture + # the old-path case for that option too, so dropping its rename entry fails + # the eval here rather than only in a real operator's config. + otelRemoteAuthelia = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = false; + # Where that elsewhere IS. Running no IdP does not mean knowing no IdP: + # the authenticator this fixture exists to render puts this address in its + # `token_url`, so a hive with a secret and no URL has a credential it can + # present nowhere. + swarm.authelia.url = "https://auth.example.invalid"; + swarm.otel.clientSecretFile = "/var/lib/swarm-otel-oidc/by-hand.secret"; + }; + + # A collector whose ONLY scrape work is published: loopback targets forced + # empty, one published job declared. Unreachable in a real deploy today — + # the module seeds `scrapeTargets.collector` under its own `enable`, so the + # loopback set is never empty on its own — which is exactly why the arm + # below needs a fixture that takes that seeding away. `mkForce` is what + # does it, and it leaves the collector itself enabled: the state under test + # is a running collector with no self-scrape, not an absent one. + otelOnlyPublished = hive { + deploy.swarm-otel.enable = true; + swarm.otel.scrapeTargets = lib.mkForce { }; + swarm.otel.publishedScrapeTargets.remote = "https://remote.t.local/metrics"; + }; + + # Two hives in the roster, which no other fixture here has: every one of + # them declares `swarm.hives.h1` alone, so a per-hive arm written against + # one of those passes on a hardcoded literal. + otelTwoHives = hive { + deploy.swarm-otel.enable = true; + deploy.authelia.enable = true; + swarm.hives.h2.domain = "h2.t.local"; + }; + cases = [ + { + # The collector's non-assertion, the deliberate mirror of Grafana's + # assertion two cases up: a host with no store identity is a supported, + # merely degraded shape here, so the reading unit simply does not exist + # rather than refusing the build. `haveCollectorSecret` is what the + # degrade already reads, unchanged by this slice. + name = "a collector with no store identity renders no reading unit, and is not refused"; + ok = + !(otelNoIdentity.systemd.services ? swarm-bao-otel-oidc) + && otelNoIdentity.services.hyperhive.deploy.swarm-otel.clientSecretFile == null + && !(lib.any (a: !a.assertion) otelNoIdentity.assertions); + } + { + # The collector's half of the same split, and a different arm from the + # authenticator case below: this one reads the PATH the unit loads, so a + # reader left on a source that is non-null but wrong still fails. The + # fixture spells the option its pre-rename way, so it covers the rename + # entry at the same time. + name = "a config written against the pre-rename otel secret path still loads it as a credential"; + ok = + lib.any (c: lib.hasInfix "/var/lib/swarm-otel-oidc/by-hand.secret" c) + otelRemoteAuthelia.containers.swarm-otel.config.systemd.services.opentelemetry-collector.serviceConfig.LoadCredential; + } + { + # The collector authenticates because it HOLDS a credential, not because + # authelia happens to share its host. Gating on the other service's + # placement renders a collector that pushes unauthenticated wherever + # authelia lives elsewhere — one of the supported shapes. + name = "a collector with a hand-delivered secret authenticates without authelia beside it"; + ok = + let + s = otelSettings otelRemoteAuthelia; + in + (s.exporters."otlphttp/victoriametrics" ? auth) + && builtins.elem "oauth2client/victoriametrics" s.service.extensions; + } + { + # Defining a receiver and attaching it are two separate lists, and the + # two gates were spelled differently: the receiver appeared for either + # scrape option, the pipeline only for the loopback one. A published- + # only collector therefore rendered scrape configs that reached no + # pipeline — requested, parsed, delivered nowhere, and valid enough to + # deploy. The receiver clause is what stops the arm passing for the + # wrong reason, by an empty `prometheus` never rendering at all. + name = "a published-only collector attaches its prometheus receiver to the swarm pipeline"; + ok = + let + s = otelSettings otelOnlyPublished; + in + otelOnlyPublished.services.hyperhive.swarm.otel.scrapeTargets == { } + && otelOnlyPublished.services.hyperhive.swarm.otel.publishedScrapeTargets != { } + && (s.receivers ? prometheus) + && builtins.elem "prometheus" s.service.pipelines."metrics/swarm".receivers; + } + { + # Read against the roster the fixture declares rather than against + # names spelled here: an arm naming `h1` passes on a single-hive + # config however the mapping is written. The length clause is what + # makes the `all` mean anything — over an empty roster it holds + # vacuously. + name = "the swarm collector routes every hive's logs, not just one"; + ok = + let + p = (otelSettings otelTwoHives).service.pipelines; + hives = lib.attrNames otelTwoHives.services.hyperhive.swarm.hives; + in + lib.length hives == 2 + && lib.all (h: (p ? "logs/${h}") && p."logs/${h}".receivers == [ "otlp/${h}" ]) hives; + } + { + # The same split as the metrics case above — defining an exporter and + # naming it are two lists — plus the half one shared list cannot have: + # the metrics store's exporter renders perfectly well inside a logs + # pipeline and posts journal records at an ingest route that is not + # for them. + name = "every logs pipeline sends to the log store and to no metrics one"; + ok = + let + s = otelSettings otelTwoHives; + logPipes = lib.filterAttrs (n: _: lib.hasPrefix "logs/" n) s.service.pipelines; + used = lib.unique (lib.concatMap (p: p.exporters) (lib.attrValues logPipes)); + in + logPipes != { } + && builtins.elem "otlphttp/victorialogs" used + && !(builtins.elem "otlphttp/victoriametrics" used) + && lib.all (e: s.exporters ? ${e}) used; + } + ]; +in +runGroup "swarm-otel-identity" cases diff --git a/nix/module-eval/swarm-services-switch.nix b/nix/module-eval/swarm-services-switch.nix new file mode 100644 index 00000000..6b2fab94 --- /dev/null +++ b/nix/module-eval/swarm-services-switch.nix @@ -0,0 +1,128 @@ +# `checks.module-eval-swarm-services-switch` — see ./lib.nix for the shared +# rationale (why this suite exists, naming convention, "evaluates +# not executes"). +{ + pkgs, + lib, + self, + nixosSystem, +}: +let + inherit + (import ./lib.nix { + inherit + pkgs + lib + self + nixosSystem + ; + }) + hive + runGroup + bridgePortsOrNone + swarmServiceEnables + ; + + # The swarm-services toggle with the central one off, so the only thing + # that can enable the gateway/resolver/bridge here is that toggle's own + # module — every other module that asserts them is behind `enable`. + swarmServicesOnly = hive { + enable = false; + deploy.allSwarmServices = true; + }; + + # Ports asked for on such a host. The request is the whole condition: the + # firewall hole exists because an operator named a port, not because the + # hive is running, and an agent reaching a host service is a claim about + # the host's own listeners either way. + exposedPortsNoHive = hive { + enable = false; + network.exposeHostPorts = [ 5432 ]; + }; + + # The swarm UI where its own toggle is on — the control the absence arm + # needs, since nothing else in this suite renders this vhost and an arm + # saying "it is not there" would hold just as well if it were never there. + # Package stubbed per this file's header: the vhost roots at it. + swarmUiHere = hive { + deploy.swarm-ui.enable = true; + deploy.swarm-ui.package = pkgs.emptyDirectory; + }; + + # The swarm's shared services hosted HERE without the all-local mode — a + # services box with hives elsewhere, the shape ./host-modules/ + # swarm-required-services.nix documents the switch for. + swarmServicesHere = hive { deploy.allSwarmServices = true; }; + + # Same, with one of those services placed on another host. Every derivation + # in that module is `mkDefault` so this stays expressible. + swarmServicesBaoElsewhere = hive { + deploy.allSwarmServices = true; + deploy.bao.enable = false; + }; + cases = [ + { + # `lib.all` over an empty set holds vacuously, so the roster is counted + # before it is read: a helper that lost a member would otherwise turn + # this case green by measuring nothing. + name = "hosting the swarm's shared services turns on every service that switch owns"; + ok = + let + es = swarmServiceEnables swarmServicesHere; + in + lib.length (lib.attrNames es) == 9 && lib.all lib.id (lib.attrValues es); + } + { + # The switch fills in for an operator who has not spoken and yields to + # one who has — that is what keeps a shared service placeable on a host + # of its own. A plain assignment or `mkForce` would satisfy both cases + # above and break this one. `nats` is the control: without it the case + # also passes on a fixture where nothing came on at all. + name = "placing one shared service elsewhere survives the switch that would enable it"; + ok = + let + es = swarmServiceEnables swarmServicesBaoElsewhere; + in + !es.bao && es.nats; + } + { + # The controller sits on the OTHER tier: `singleHostSwarm` places it + # (./host-modules/local-defaults.nix) and swarm-ui follows the + # controller. Pinned so that moving a service between tiers is a + # decision someone makes rather than a merge nobody reads. + name = "hosting the swarm's shared services does not make a hive the swarm's control plane"; + ok = + !swarmServicesHere.services.hyperhive.deploy.swarm-controller.enable + && !swarmServicesHere.services.hyperhive.deploy.swarm-ui.enable; + } + { + # Control for the arm above, and the one place this slice is not inert: + # the ports are opened because they were named, on a host that never + # turned the hive on. The bridge firewall is the host's own, so there is + # nothing here for the hive toggle to have been protecting. + name = "a named exposeHostPorts opens its bridge port on the host's own say-so"; + ok = builtins.elem 5432 (bridgePortsOrNone exposedPortsNoHive); + } + { + name = "the swarm UI claims the swarm apex where this host serves it"; + ok = swarmUiHere.services.nginx.virtualHosts ? "t.local"; + } + { + # The swarm-services toggle enables them explicitly, from its own + # module rather than from any of their defaults. + name = "the swarm-services toggle turns on the gateway, resolver and bridge by itself"; + ok = + swarmServicesOnly.services.hyperhive.gateway.enable + && swarmServicesOnly.services.hyperhive.gateway.dns.enable + && swarmServicesOnly.services.hyperhive.network.enable; + } + { + # An operator's explicit `false` beats every `mkDefault` assertion, + # which is what keeps "asserted by whoever needs it" from being a + # setting the operator cannot turn off. + name = "an explicit gateway.enable = false wins over the modules asserting it"; + ok = !(hive { gateway.enable = false; }).services.nginx.enable; + } + ]; +in +runGroup "swarm-services-switch" cases